From 9e99ffe50aa6beb6c23aec0f3bbbc11d3d0c81e8 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 12:31:43 -0500 Subject: [PATCH 01/43] initial push --- smartsim/settingshold/__init__.py | 35 ++ smartsim/settingshold/baseSettings.py | 35 ++ smartsim/settingshold/batchCommand.py | 6 + smartsim/settingshold/batchSettings.py | 205 ++++++++++ smartsim/settingshold/common.py | 13 + smartsim/settingshold/launchCommand.py | 12 + smartsim/settingshold/launchSettings.py | 349 ++++++++++++++++++ smartsim/settingshold/translators/__init__.py | 7 + .../translators/batch/__init.__.py | 9 + .../settingshold/translators/batch/lsf.py | 118 ++++++ .../settingshold/translators/batch/pbs.py | 167 +++++++++ .../settingshold/translators/batch/slurm.py | 105 ++++++ .../translators/batchArgTranslator.py | 134 +++++++ .../translators/launch/__init__.py | 19 + .../settingshold/translators/launch/alps.py | 162 ++++++++ .../settingshold/translators/launch/dragon.py | 31 ++ .../settingshold/translators/launch/local.py | 41 ++ .../settingshold/translators/launch/lsf.py | 71 ++++ .../settingshold/translators/launch/mpi.py | 156 ++++++++ .../settingshold/translators/launch/pals.py | 106 ++++++ .../settingshold/translators/launch/slurm.py | 244 ++++++++++++ .../translators/launchArgTranslator.py | 178 +++++++++ .../settings_test/test_alpsLauncher.py | 94 +++++ .../settings_test/test_aprunLauncher.py | 138 +++++++ .../settings_test/test_dragonLauncher.py | 65 ++++ .../settings_test/test_localLauncher.py | 157 ++++++++ .../settings_test/test_lsfLauncher.py | 96 +++++ .../settings_test/test_lsfScheduler.py | 31 ++ .../settings_test/test_mpiLauncher.py | 162 ++++++++ .../settings_test/test_palsLauncher.py | 79 ++++ .../settings_test/test_pbsScheduler.py | 37 ++ .../settings_test/test_slurmLauncher.py | 187 ++++++++++ .../settings_test/test_slurmScheduler.py | 109 ++++++ tests/temp_tests/settings_tests.py | 123 ------ 34 files changed, 3358 insertions(+), 123 deletions(-) create mode 100644 smartsim/settingshold/__init__.py create mode 100644 smartsim/settingshold/baseSettings.py create mode 100644 smartsim/settingshold/batchCommand.py create mode 100644 smartsim/settingshold/batchSettings.py create mode 100644 smartsim/settingshold/common.py create mode 100644 smartsim/settingshold/launchCommand.py create mode 100644 smartsim/settingshold/launchSettings.py create mode 100644 smartsim/settingshold/translators/__init__.py create mode 100644 smartsim/settingshold/translators/batch/__init.__.py create mode 100644 smartsim/settingshold/translators/batch/lsf.py create mode 100644 smartsim/settingshold/translators/batch/pbs.py create mode 100644 smartsim/settingshold/translators/batch/slurm.py create mode 100644 smartsim/settingshold/translators/batchArgTranslator.py create mode 100644 smartsim/settingshold/translators/launch/__init__.py create mode 100644 smartsim/settingshold/translators/launch/alps.py create mode 100644 smartsim/settingshold/translators/launch/dragon.py create mode 100644 smartsim/settingshold/translators/launch/local.py create mode 100644 smartsim/settingshold/translators/launch/lsf.py create mode 100644 smartsim/settingshold/translators/launch/mpi.py create mode 100644 smartsim/settingshold/translators/launch/pals.py create mode 100644 smartsim/settingshold/translators/launch/slurm.py create mode 100644 smartsim/settingshold/translators/launchArgTranslator.py create mode 100644 tests/temp_tests/settings_test/test_alpsLauncher.py create mode 100644 tests/temp_tests/settings_test/test_aprunLauncher.py create mode 100644 tests/temp_tests/settings_test/test_dragonLauncher.py create mode 100644 tests/temp_tests/settings_test/test_localLauncher.py create mode 100644 tests/temp_tests/settings_test/test_lsfLauncher.py create mode 100644 tests/temp_tests/settings_test/test_lsfScheduler.py create mode 100644 tests/temp_tests/settings_test/test_mpiLauncher.py create mode 100644 tests/temp_tests/settings_test/test_palsLauncher.py create mode 100644 tests/temp_tests/settings_test/test_pbsScheduler.py create mode 100644 tests/temp_tests/settings_test/test_slurmLauncher.py create mode 100644 tests/temp_tests/settings_test/test_slurmScheduler.py delete mode 100644 tests/temp_tests/settings_tests.py diff --git a/smartsim/settingshold/__init__.py b/smartsim/settingshold/__init__.py new file mode 100644 index 0000000000..41076efd3e --- /dev/null +++ b/smartsim/settingshold/__init__.py @@ -0,0 +1,35 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .launchSettings import LaunchSettings +from .baseSettings import BaseSettings +from .batchSettings import BatchSettings + +__all__ = [ + "LaunchSettings", + "BaseSettings", + "BatchSettings" +] \ No newline at end of file diff --git a/smartsim/settingshold/baseSettings.py b/smartsim/settingshold/baseSettings.py new file mode 100644 index 0000000000..3cc5d71076 --- /dev/null +++ b/smartsim/settingshold/baseSettings.py @@ -0,0 +1,35 @@ +# BSD 2-Clause License # +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import annotations + +import typing as t +from ..log import get_logger + +logger = get_logger(__name__) + +# fmt: off +class BaseSettings: + ... +# fmt: on \ No newline at end of file diff --git a/smartsim/settingshold/batchCommand.py b/smartsim/settingshold/batchCommand.py new file mode 100644 index 0000000000..b9c43a6acd --- /dev/null +++ b/smartsim/settingshold/batchCommand.py @@ -0,0 +1,6 @@ +from enum import Enum + +class SchedulerType(Enum): + SlurmLauncher = "sbatch" + PbsLauncher = "qsub" + LsfLauncher = "bsub" \ No newline at end of file diff --git a/smartsim/settingshold/batchSettings.py b/smartsim/settingshold/batchSettings.py new file mode 100644 index 0000000000..b31325e8ec --- /dev/null +++ b/smartsim/settingshold/batchSettings.py @@ -0,0 +1,205 @@ +from __future__ import annotations +from enum import Enum +import typing as t +import copy + +from smartsim.log import get_logger + +from .translators.batch.pbs import QsubBatchArgTranslator +from .translators.batch.slurm import SlurmBatchArgTranslator +from .translators.batch.lsf import BsubBatchArgTranslator +from .translators import BatchArgTranslator + +IntegerArgument_1 = t.Dict[str, t.Optional[int]] +FloatArgument_1 = t.Dict[str, t.Optional[float]] +StringArgument_1 = t.Dict[str, t.Optional[str]] + +logger = get_logger(__name__) + +class SupportedLaunchers(Enum): + """ Launchers that are supported by + SmartSim. + """ + pbs = "qsub" + lsf = "bsub" + slurm = "sbatch" + +class BatchSettings(): + def __init__( + self, + scheduler: str, + scheduler_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, + env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, + **kwargs: t.Any, + ) -> None: + scheduler_to_translator = { + 'sbatch' : SlurmBatchArgTranslator(), + 'jsrun' : BsubBatchArgTranslator(), + 'qsub' : QsubBatchArgTranslator(), + } + if scheduler in scheduler_to_translator: + self.scheduler = scheduler + else: + raise ValueError(f"'{scheduler}' is not a valid scheduler name.") + + # TODO check and preprocess env_vars + self.env_vars = env_vars or {} + + # TODO check and preporcess launcher_args + self.scheduler_args = scheduler_args or {} + self.arg_translator = t.cast(BatchArgTranslator,scheduler_to_translator.get(self.scheduler)) + + @property + def scheduler_args(self) -> t.Dict[str, t.Optional[str]]: + """Retrieve attached batch arguments + + :returns: attached batch arguments + """ + return self._scheduler_args + + @scheduler_args.setter + def scheduler_args(self, value: t.Dict[str, t.Optional[str]]) -> None: + """Attach batch arguments + + :param value: dictionary of batch arguments + """ + self._scheduler_args = copy.deepcopy(value) if value else {} + + def set_walltime(self, walltime: str) -> None: + """Set the walltime of the job + + format = "HH:MM:SS" + + :param walltime: wall time + """ + # TODO check for formatting here + args = self.arg_translator.set_walltime(walltime) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_nodes(self, num_nodes: int) -> None: + """Set the number of nodes for this batch job + + :param num_nodes: number of nodes + """ + args = self.arg_translator.set_nodes(num_nodes) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_account(self, account: str) -> None: + """Set the account for this batch job + + :param account: account id + """ + args = self.arg_translator.set_account(account) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_partition(self, partition: str) -> None: + """Set the partition for the batch job + + :param partition: partition name + """ + args = self.arg_translator.set_partition(partition) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_queue(self, queue: str) -> None: + """alias for set_partition + + Sets the partition for the slurm batch job + + :param queue: the partition to run the batch job on + """ + args = self.arg_translator.set_queue(queue) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_cpus_per_task(self, cpus_per_task: int) -> None: + """Set the number of cpus to use per task + + This sets ``--cpus-per-task`` + + :param num_cpus: number of cpus to use per task + """ + args = self.arg_translator.set_cpus_per_task(cpus_per_task) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: + """Specify the hostlist for this job + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + args = self.arg_translator.set_hostlist(host_list) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_smts(self, smts: int) -> None: + """Set SMTs + + This sets ``-alloc_flags``. If the user sets + SMT explicitly through ``-alloc_flags``, then that + takes precedence. + + :param smts: SMT (e.g on Summit: 1, 2, or 4) + """ + args = self.arg_translator.set_smts(smts) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_project(self, project: str) -> None: + """Set the project + + This sets ``-P``. + + :param time: project name + """ + args = self.arg_translator.set_project(project) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_tasks(self, tasks: int) -> None: + """Set the number of tasks for this job + + This sets ``-n`` + + :param tasks: number of tasks + """ + args = self.arg_translator.set_tasks(tasks) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_ncpus(self, num_cpus: int) -> None: + """Set the number of cpus obtained in each node. + + If a select argument is provided in + ``QsubBatchSettings.resources``, then + this value will be overridden + + :param num_cpus: number of cpus per node in select + """ + args = self.arg_translator.set_ncpus(num_cpus) + if args: + for key, value in args.items(): + self.set(key, value) + + def format_batch_args(self) -> t.List[str]: + """Get the formatted batch arguments for a preview + """ + return self.arg_translator.format_batch_args(self.scheduler_args) + + def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: + # Store custom arguments in the launcher_args + self.scheduler_args[key] = arg \ No newline at end of file diff --git a/smartsim/settingshold/common.py b/smartsim/settingshold/common.py new file mode 100644 index 0000000000..79f1bc40ee --- /dev/null +++ b/smartsim/settingshold/common.py @@ -0,0 +1,13 @@ +import typing as t + +IntegerArgument = t.Dict[str, t.Optional[int]] +FloatArgument = t.Dict[str, t.Optional[float]] +StringArgument = t.Dict[str, t.Optional[str]] + +def process_env_vars(env_vars: StringArgument): + for key, value in env_vars.items(): + if not isinstance(value, str): + raise ValueError(f"Value for '{key}' must be a string.") + +# def process_launcher_args(launcher_args: t.Dict[str, t.Union[str,int,float,None]]): + \ No newline at end of file diff --git a/smartsim/settingshold/launchCommand.py b/smartsim/settingshold/launchCommand.py new file mode 100644 index 0000000000..fd67814673 --- /dev/null +++ b/smartsim/settingshold/launchCommand.py @@ -0,0 +1,12 @@ +from enum import Enum + +class LauncherType(Enum): + DragonLauncher = "dragon" + SlurmLauncher = "slurm" + PalsLauncher = "pals" + AlpsLauncher = "aprun" + LocalLauncher = "local" + MpiexecLauncher = "mpiexec" + MpirunLauncher = "mpirun" + OrterunLauncher = "orterun" + LsfLauncher = "jsrun" \ No newline at end of file diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py new file mode 100644 index 0000000000..3ed744b9b3 --- /dev/null +++ b/smartsim/settingshold/launchSettings.py @@ -0,0 +1,349 @@ +from __future__ import annotations +from enum import Enum +import typing as t +import copy + + +from smartsim.log import get_logger + +from .launchCommand import LauncherType +from .translators.launch.alps import AprunArgTranslator +from .translators.launch.lsf import JsrunArgTranslator +from .translators.launch.mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator +from .translators.launch.pals import PalsMpiexecArgTranslator +from .translators.launch.slurm import SlurmArgTranslator +from .translators.launch.dragon import DragonArgTranslator +from .translators.launch.local import LocalArgTranslator +from .translators import LaunchArgTranslator + +from .common import process_env_vars, IntegerArgument, StringArgument, FloatArgument + +logger = get_logger(__name__) + +class SupportedLaunchers(Enum): + """ Launchers that are supported by + SmartSim. + """ + local = "local" + dragon = "dragon" + slurm = "slurm" + mpiexec = "mpiexec" + mpirun = "mpirun" + orterun = "orterun" + aprun = "aprun" + jsrun = "jsrun" + pals = "pals" + +class LaunchSettings(): + def __init__( + self, + launcher: LauncherType, + launcher_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, + env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, + **kwargs: t.Any, + ) -> None: + launcher_to_translator : t.Dict[str,LaunchArgTranslator] = { + 'slurm' : SlurmArgTranslator(), + 'mpiexec' : MpiexecArgTranslator(), + 'mpirun' : MpiArgTranslator(), + 'orterun' : OrteArgTranslator(), + 'aprun' : AprunArgTranslator(), + 'jsrun' : JsrunArgTranslator(), + 'pals' : PalsMpiexecArgTranslator(), + 'dragon': DragonArgTranslator(), + 'local': LocalArgTranslator(), + } + if launcher in launcher_to_translator: + self.launcher = launcher + else: + raise ValueError(f"'{launcher}' is not a valid launcher name.") + + process_env_vars(env_vars) + self.env_vars = env_vars or {} + + # TODO check and preporcess launcher_args + self.launcher_args = launcher_args or {} + + self.arg_translator = t.cast(LaunchArgTranslator,launcher_to_translator.get(launcher)) + + @property + def launcher_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: + """Return an immutable list of attached run arguments. + + :returns: attached run arguments + """ + return self._launcher_args + + @launcher_args.setter + def launcher_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> None: + """Set the run arguments. + + :param value: run arguments + """ + self._launcher_args = copy.deepcopy(value) + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return self.arg_translator.launcher_str() + + def set_nodes(self, nodes: int) -> None: + """ Sets the number of nodes + + :param nodes: The number of nodes + """ + + args = self.arg_translator.set_nodes(nodes) + + if args: + for key, value in args.items(): + self.set(key, value) + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: + """ Specify the hostlist for this job + + :param host_list: hosts to launch on + """ + + args = self.arg_translator.set_hostlist(host_list) + + if args: + for key, value in args.items(): + self.set(key, value) + + def set_hostlist_from_file(self, file_path: str) -> None: + """ Use the contents of a file to set the node list + + :param file_path: Path to the hostlist file + """ + + args = self.arg_translator.set_hostlist_from_file(file_path) + + if args: + for key, value in args.items(): + self.set(key, value) + + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: + """ Specify a list of hosts to exclude for launching this job + + :param host_list: hosts to exclude + """ + + args = self.arg_translator.set_excluded_hosts(host_list) + + if args: + for key, value in args.items(): + self.set(key, value) + + def set_cpus_per_task(self, cpus_per_task: int) -> None: + """ Set the number of cpus to use per task + + :param cpus_per_task: number of cpus to use per task + """ + + args = self.arg_translator.set_cpus_per_task(cpus_per_task) + + if args: + for key, value in args.items(): + self.set(key, value) + + def set_tasks(self, tasks: int) -> None: + """ Set the number of tasks for this job + + :param tasks: number of tasks + """ + args = self.arg_translator.set_tasks(tasks) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_tasks_per_node(self, tasks_per_node: int) -> None: + """ Set the number of tasks per node for this job + + :param tasks_per_node: number of tasks per node + """ + args = self.arg_translator.set_tasks_per_node(tasks_per_node) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: + """ Bind by setting CPU masks on tasks + + :param bindings: List specifing the cores to which MPI processes are bound + """ + args = self.arg_translator.set_cpu_bindings(bindings) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_memory_per_node(self, memory_per_node: int) -> None: + """ Specify the real memory required per node + + :param memory_per_node: Amount of memory per node in megabytes + """ + args = self.arg_translator.set_memory_per_node(memory_per_node) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_executable_broadcast(self, dest_path: str) -> None: + """ Copy executable file to allocated compute nodes + + :param dest_path: Path to copy an executable file + """ + args = self.arg_translator.set_executable_broadcast(dest_path) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: + """Specify the node feature for this job + + :param feature_list: node feature to launch on + :raises TypeError: if not str or list of str + """ + args = self.arg_translator.set_node_feature(feature_list) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_walltime(self, walltime: str) -> None: + """Set the walltime of the job + + :param walltime: wall time + """ + args = self.arg_translator.set_walltime(walltime) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_binding(self, binding: str) -> None: + """Set binding + + This sets ``--bind`` + + :param binding: Binding, e.g. `packed:21` + """ + args = self.arg_translator.set_binding(binding) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_cpu_binding_type(self, bind_type: str) -> None: + """Specifies the cores to which MPI processes are bound + + This sets ``--bind-to`` for MPI compliant implementations + + :param bind_type: binding type + """ + args = self.arg_translator.set_cpu_binding_type(bind_type) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_task_map(self, task_mapping: str) -> None: + """Set ``mpirun`` task mapping + + this sets ``--map-by `` + + For examples, see the man page for ``mpirun`` + + :param task_mapping: task mapping + """ + args = self.arg_translator.set_task_map(task_mapping) + if args: + for key, value in args.items(): + self.set(key, value) + + def set_verbose_launch(self, verbose: bool) -> None: + """Set the job to run in verbose mode + + This sets ``--verbose`` + + :param verbose: Whether the job should be run verbosely + """ + args = self.arg_translator.set_verbose_launch(verbose) + if args and verbose: + for key, value in args.items(): + self.set(key, value) + if args and not verbose: + self.launcher_args.pop(next(iter(args))) + + def set_quiet_launch(self, quiet: bool) -> None: + """Set the job to run in quiet mode + + This sets ``--quiet`` + + :param quiet: Whether the job should be run quietly + """ + args = self.arg_translator.set_quiet_launch(quiet) + if args and quiet: + for key, value in args.items(): + self.set(key, value) + if args and not quiet: + self.launcher_args.pop(next(iter(args))) + + def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: + """Build environment variable string for Slurm + + Slurm takes exports in comma separated lists + the list starts with all as to not disturb the rest of the environment + for more information on this, see the slurm documentation for srun + + :returns: the formatted string of environment variables + """ + return self.arg_translator.format_comma_sep_env_vars(self.env_vars) + + def format_launcher_args(self) -> t.Union[t.List[str],None]: + """Return formatted launch arguments + + For ``RunSettings``, the run arguments are passed + literally with no formatting. + + :return: list run arguments for these settings + """ + return self.arg_translator.format_launcher_args(self.launcher_args) + + def format_env_vars(self) -> t.Union[t.List[str],None]: + """Build bash compatible environment variable string for Slurm + + :returns: the formatted string of environment variables + """ + return self.arg_translator.format_env_vars(self.env_vars) + + def update_env(self, env_vars: t.Dict[str, t.Union[str, int, float, bool]]) -> None: + """Update the job environment variables + + To fully inherit the current user environment, add the + workload-manager-specific flag to the launch command through the + :meth:`add_exe_args` method. For example, ``--export=ALL`` for + slurm, or ``-V`` for PBS/aprun. + + + :param env_vars: environment variables to update or add + :raises TypeError: if env_vars values cannot be coerced to strings + """ + val_types = (str, int, float, bool) + # Coerce env_vars values to str as a convenience to user + for env, val in env_vars.items(): + if not isinstance(val, val_types): + raise TypeError( + f"env_vars[{env}] was of type {type(val)}, not {val_types}" + ) + + self.env_vars[env] = str(val) + + def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: + # Store custom arguments in the launcher_args + if not isinstance(key, str): + raise TypeError("Argument name should be of type str") + # if value is not None and not isinstance(value, str): + # raise TypeError("Argument value should be of type str or None") + # arg = arg.strip().lstrip("-") + # if not condition: + # logger.info(f"Could not set argument '{arg}': condition not met") + # return + if key in self.launcher_args and arg != self.launcher_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{arg}'") + self.launcher_args[key] = arg \ No newline at end of file diff --git a/smartsim/settingshold/translators/__init__.py b/smartsim/settingshold/translators/__init__.py new file mode 100644 index 0000000000..af6351065f --- /dev/null +++ b/smartsim/settingshold/translators/__init__.py @@ -0,0 +1,7 @@ +from .launchArgTranslator import LaunchArgTranslator +from .batchArgTranslator import BatchArgTranslator + +__all__ = [ + "LaunchArgTranslator", + "BatchArgTranslator" +] \ No newline at end of file diff --git a/smartsim/settingshold/translators/batch/__init.__.py b/smartsim/settingshold/translators/batch/__init.__.py new file mode 100644 index 0000000000..6416819f85 --- /dev/null +++ b/smartsim/settingshold/translators/batch/__init.__.py @@ -0,0 +1,9 @@ +# from .lsf import BsubBatchArgTranslator +# from .pbs import QsubBatchArgTranslator +# from .slurm import SlurmBatchArgTranslator + +# __all__ = [ +# "BsubBatchArgTranslator", +# "QsubBatchArgTranslator", +# "SlurmBatchArgTranslator", +# ] \ No newline at end of file diff --git a/smartsim/settingshold/translators/batch/lsf.py b/smartsim/settingshold/translators/batch/lsf.py new file mode 100644 index 0000000000..c74cb759f3 --- /dev/null +++ b/smartsim/settingshold/translators/batch/lsf.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +from ..batchArgTranslator import BatchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from smartsim.log import get_logger +from ...batchCommand import SchedulerType + +logger = get_logger(__name__) + +class BsubBatchArgTranslator(BatchArgTranslator): + + def scheduler_str(self) -> str: + """ Get the string representation of the launcher + """ + return SchedulerType.LsfLauncher.value + + def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + """Set the walltime + + This sets ``-W``. + + :param walltime: Time in hh:mm format, e.g. "10:00" for 10 hours, + if time is supplied in hh:mm:ss format, seconds + will be ignored and walltime will be set as ``hh:mm`` + """ + # For compatibility with other launchers, as explained in docstring + if walltime: + if len(walltime.split(":")) > 2: + walltime = ":".join(walltime.split(":")[:2]) + return {"W": walltime} + + def set_smts(self, smts: int) -> t.Union[IntegerArgument,None]: + """Set SMTs + + This sets ``-alloc_flags``. If the user sets + SMT explicitly through ``-alloc_flags``, then that + takes precedence. + + :param smts: SMT (e.g on Summit: 1, 2, or 4) + """ + return {"alloc_flags": int(smts)} + + def set_project(self, project: str) -> t.Union[StringArgument,None]: + """Set the project + + This sets ``-P``. + + :param time: project name + """ + return {"P": project} + + def set_account(self, account: str) -> t.Union[StringArgument,None]: + """Set the project + + this function is an alias for `set_project`. + + :param account: project name + """ + return self.set_project(account) + + def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + """Set the number of nodes for this batch job + + This sets ``-nnodes``. + + :param nodes: number of nodes + """ + return {"nnodes": int(num_nodes)} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """Specify the hostlist for this job + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"m": '"' + " ".join(host_list) + '"'} + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + """Set the number of tasks for this job + + This sets ``-n`` + + :param tasks: number of tasks + """ + return {"n": int(tasks)} + + def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + """Set the queue for this job + + :param queue: The queue to submit the job on + """ + return {"q": queue} + + def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + """Get the formatted batch arguments for a preview + + :return: list of batch arguments for Qsub + """ + opts = [] + + for opt, value in batch_args.items(): + + prefix = "-" # LSF only uses single dashses + + if not value: + opts += [prefix + opt] + else: + opts += [" ".join((prefix + opt, str(value)))] + + return opts \ No newline at end of file diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py new file mode 100644 index 0000000000..5a9935537c --- /dev/null +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +from enum import Enum +from copy import deepcopy +import typing as t +from ..batchArgTranslator import BatchArgTranslator +from ....error import SSConfigError +from ...common import IntegerArgument, StringArgument, FloatArgument +from smartsim.log import get_logger +from ...batchCommand import SchedulerType +logger = get_logger(__name__) + +class QsubBatchArgTranslator(BatchArgTranslator): + + def scheduler_str(self) -> str: + """ Get the string representation of the launcher + """ + return SchedulerType.PbsLauncher.value + + def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + """Set the number of nodes for this batch job + + In PBS, 'select' is the more primitive way of describing how + many nodes to allocate for the job. 'nodes' is equivalent to + 'select' with a 'place' statement. Assuming that only advanced + users would use 'set_resource' instead, defining the number of + nodes here is sets the 'nodes' resource. + + :param num_nodes: number of nodes + """ + + return {"nodes": num_nodes} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """Specify the hostlist for this job + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be a list of strings") + return {"hostname": ",".join(host_list)} + + def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + """Set the walltime of the job + + format = "HH:MM:SS" + + If a walltime argument is provided in + ``QsubBatchSettings.resources``, then + this value will be overridden + + :param walltime: wall time + """ + return {"walltime": walltime} + + def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + """Set the queue for the batch job + + :param queue: queue name + """ + return {"q": str(queue)} + + def set_ncpus(self, num_cpus: int) -> t.Union[IntegerArgument,None]: + """Set the number of cpus obtained in each node. + + If a select argument is provided in + ``QsubBatchSettings.resources``, then + this value will be overridden + + :param num_cpus: number of cpus per node in select + """ + return {"ppn": int(num_cpus)} + + def set_account(self, account: str) -> t.Union[StringArgument,None]: + """Set the account for this batch job + + :param acct: account id + """ + return {"A": str(account)} + + def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + """Get the formatted batch arguments for a preview + + :return: batch arguments for Qsub + :raises ValueError: if options are supplied without values + """ + opts, batch_arg_copy = self._create_resource_list(batch_args) + t.cast(t.List,batch_arg_copy) + for opt, value in batch_arg_copy.items(): + prefix = "-" + if not value: + raise ValueError("PBS options without values are not allowed") + opts += [" ".join((prefix + opt, str(value)))] + return opts + + def _sanity_check_resources( + self, batch_args: t.Dict[str, t.Union[str,int,float,None]] + ) -> None: + """Check that only select or nodes was specified in resources + + Note: For PBS Pro, nodes is equivalent to 'select' and 'place' so + they are not quite synonyms. Here we assume that + """ + # Note: isinstance check here to avoid collision with default + checked_resources = batch_args + + has_select = checked_resources.get("select", None) + has_nodes = checked_resources.get("nodes", None) + + if has_select and has_nodes: + raise SSConfigError( + "'select' and 'nodes' cannot both be specified. This can happen " + "if nodes were specified using the 'set_nodes' method and " + "'select' was set using 'set_resource'. Please only specify one." + ) + + if has_select and not isinstance(has_select, int): + raise TypeError("The value for 'select' must be an integer") + if has_nodes and not isinstance(has_nodes, int): + raise TypeError("The value for 'nodes' must be an integer") + + for key, value in checked_resources.items(): + if not isinstance(key, str): + raise TypeError( + f"The type of {key=} is {type(key)}. Only int and str " + "are allowed." + ) + if not isinstance(value, (str, int)): + raise TypeError( + f"The value associated with {key=} is {type(value)}. Only int " + "and str are allowed." + ) + + def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + self._sanity_check_resources(batch_args) + res = [] + + batch_arg_copy = deepcopy(batch_args) + # Construct the basic select/nodes statement + if select := batch_arg_copy.pop("select", None): + select_command = f"-l select={select}" + elif nodes := batch_arg_copy.pop("nodes", None): + select_command = f"-l nodes={nodes}" + else: + raise SSConfigError( + "Insufficient resource specification: no nodes or select statement" + ) + if ncpus := batch_arg_copy.pop("ppn", None): + select_command += f":ncpus={ncpus}" + if hosts := batch_arg_copy.pop("hostname", None): + hosts_list = ["=".join(hosts)] + t.cast(str,hosts_list) + select_command += f":{'+'.join(hosts_list)}" + res += [select_command] + if walltime := batch_arg_copy.pop("walltime", None): + res += [f"-l walltime={walltime}"] + + # All other "standard" resource specs + # for resource, value in batch_arg_copy.items(): + # res += [f"-l {resource}={value}"] + + return res, batch_arg_copy \ No newline at end of file diff --git a/smartsim/settingshold/translators/batch/slurm.py b/smartsim/settingshold/translators/batch/slurm.py new file mode 100644 index 0000000000..d1985c5e4e --- /dev/null +++ b/smartsim/settingshold/translators/batch/slurm.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from enum import Enum +import re +import typing as t +from ..batchArgTranslator import BatchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...batchCommand import SchedulerType +from smartsim.log import get_logger + +logger = get_logger(__name__) + +class SlurmBatchArgTranslator(BatchArgTranslator): + + def scheduler_str(self) -> str: + """ Get the string representation of the scheduler + """ + return SchedulerType.SlurmLauncher.value + + def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + """Set the walltime of the job + + format = "HH:MM:SS" + + :param walltime: wall time + """ + pattern = r'^\d{2}:\d{2}:\d{2}$' + if walltime and re.match(pattern, walltime): + return {"time": str(walltime)} + else: + raise ValueError("Invalid walltime format. Please use 'HH:MM:SS' format.") + + def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + """Set the number of nodes for this batch job + + :param num_nodes: number of nodes + """ + return {"nodes": int(num_nodes)} + + def set_account(self, account: str) -> t.Union[StringArgument,None]: + """Set the account for this batch job + + :param account: account id + """ + return {"account": account} + + def set_partition(self, partition: str) -> t.Union[StringArgument,None]: + """Set the partition for the batch job + + :param partition: partition name + """ + return {"partition": str(partition)} + + def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + """alias for set_partition + + Sets the partition for the slurm batch job + + :param queue: the partition to run the batch job on + """ + return self.set_partition(queue) + + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: + """Set the number of cpus to use per task + + This sets ``--cpus-per-task`` + + :param num_cpus: number of cpus to use per task + """ + return {"cpus-per-task": int(cpus_per_task)} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """Specify the hostlist for this job + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"nodelist": ",".join(host_list)} + + def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + """Get the formatted batch arguments for a preview + + :return: batch arguments for Sbatch + """ + opts = [] + # TODO add restricted here + for opt, value in batch_args.items(): + # attach "-" prefix if argument is 1 character otherwise "--" + short_arg = bool(len(str(opt)) == 1) + prefix = "-" if short_arg else "--" + + if not value: + opts += [prefix + opt] + else: + if short_arg: + opts += [prefix + opt, str(value)] + else: + opts += ["=".join((prefix + opt, str(value)))] + return opts \ No newline at end of file diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py new file mode 100644 index 0000000000..1dbea70712 --- /dev/null +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +import typing as t +# from launchcommand import LauncherType +from ..common import IntegerArgument, StringArgument, FloatArgument + +from smartsim.log import get_logger + +logger = get_logger(__name__) + +from ..common import IntegerArgument, StringArgument, FloatArgument + +class BatchArgTranslator(ABC): #ArgTranslator - LaunchArgTranslatro and BatchArgTranslator + """Abstract base class that defines all generic launcher + argument methods that are not supported. It is the + responsibility of child classes for each launcher to translate + the input parameter to a properly formatted launcher argument. + """ + + @abstractmethod + def scheduler_str(self) -> str: + """ Get the string representation of the launcher + """ + pass + + def set_account(self, account: str) -> t.Union[StringArgument,None]: + """Set the account for this batch job + + :param account: account id + """ + logger.warning(f"set_account() not supported for {self.scheduler_str()}.") + return None + + def set_partition(self, partition: str) -> t.Union[StringArgument,None]: + """Set the partition for the batch job + + :param partition: partition name + """ + logger.warning(f"set_partition() not supported for {self.scheduler_str()}.") + return None + + def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + """alias for set_partition + + Sets the partition for the slurm batch job + + :param queue: the partition to run the batch job on + """ + logger.warning(f"set_queue() not supported for {self.scheduler_str()}.") + return None + + def set_smts(self, smts: int) -> t.Union[IntegerArgument,None]: + """Set SMTs + + This sets ``-alloc_flags``. If the user sets + SMT explicitly through ``-alloc_flags``, then that + takes precedence. + + :param smts: SMT (e.g on Summit: 1, 2, or 4) + """ + logger.warning(f"set_smts() not supported for {self.scheduler_str()}.") + return None + + def set_project(self, project: str) -> t.Union[StringArgument,None]: + """Set the project + + This sets ``-P``. + + :param time: project name + """ + logger.warning(f"set_project() not supported for {self.scheduler_str()}.") + return None + + def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + """Set the walltime of the job + + :param walltime: wall time + """ + logger.warning(f"set_walltime() not supported for {self.scheduler_str()}.") + return None + + def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + """Set the number of nodes for this batch job + + :param num_nodes: number of nodes + """ + logger.warning(f"set_nodes() not supported for {self.scheduler_str()}.") + return None + + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: + """Set the number of cpus to use per task + + :param num_cpus: number of cpus to use per task + """ + logger.warning(f"set_cpus_per_task() not supported for {self.scheduler_str()}.") + return None + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """Specify the hostlist for this job + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + logger.warning(f"set_hostlist() not supported for {self.scheduler_str()}.") + return None + + def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + """Get the formatted batch arguments for a preview + + :return: batch arguments for Sbatch + """ + logger.warning(f"format_batch_args() not supported for {self.scheduler_str()}.") + return None + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + """Set the number of tasks for this job + + :param tasks: number of tasks + """ + logger.warning(f"set_tasks() not supported for {self.scheduler_str()}.") + return None + + def set_ncpus(self, num_cpus: int) -> t.Union[IntegerArgument,None]: + """Set the number of cpus obtained in each node. + + If a select argument is provided in + ``QsubBatchSettings.resources``, then + this value will be overridden + + :param num_cpus: number of cpus per node in select + """ + logger.warning(f"set_ncpus() not supported for {self.scheduler_str()}.") + return None \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/__init__.py b/smartsim/settingshold/translators/launch/__init__.py new file mode 100644 index 0000000000..3eba5e05db --- /dev/null +++ b/smartsim/settingshold/translators/launch/__init__.py @@ -0,0 +1,19 @@ +from .alps import AprunArgTranslator +from .dragon import DragonArgTranslator +from .local import LocalArgTranslator +from .lsf import JsrunArgTranslator +from .mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator +from .pals import PalsMpiexecArgTranslator +from .slurm import SlurmArgTranslator + +__all__ = [ + "AprunArgTranslator", + "DragonArgTranslator", + "LocalArgTranslator", + "JsrunArgTranslator", + "MpiArgTranslator", + "MpiexecArgTranslator", + "OrteArgTranslator", + "PalsMpiexecArgTranslator", + "SlurmArgTranslator", +] \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py new file mode 100644 index 0000000000..8a451d7804 --- /dev/null +++ b/smartsim/settingshold/translators/launch/alps.py @@ -0,0 +1,162 @@ +from ..launchArgTranslator import LaunchArgTranslator +import typing as t +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...launchCommand import LauncherType +from smartsim.log import get_logger + +logger = get_logger(__name__) + +class AprunArgTranslator(LaunchArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.AlpsLauncher.value + + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: + """Set the number of cpus to use per task + + This sets ``--cpus-per-pe`` + + :param cpus_per_task: number of cpus to use per task + """ + return {"--cpus-per-pe": int(cpus_per_task)} + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + """Set the number of tasks for this job + + This sets ``--pes`` + + :param tasks: number of tasks + """ + return {"--pes": int(tasks)} + + def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + """Set the number of tasks for this job + + This sets ``--pes-per-node`` + + :param tasks_per_node: number of tasks per node + """ + return {"--pes-per-node": int(tasks_per_node)} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + """Specify the hostlist for this job + + This sets ``--node-list`` + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"--node-list": ",".join(host_list)} + + def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: + """Use the contents of a file to set the node list + + This sets ``--node-list-file`` + + :param file_path: Path to the hostlist file + """ + return {"--node-list-file": file_path} + + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + """Specify a list of hosts to exclude for launching this job + + This sets ``--exclude-node-list`` + + :param host_list: hosts to exclude + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"--exclude-node-list": ",".join(host_list)} + + def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> t.Union[StringArgument, None]: + """Specifies the cores to which MPI processes are bound + + This sets ``--cpu-binding`` + + :param bindings: List of cpu numbers + """ + if isinstance(bindings, int): + bindings = [bindings] + return {"--cpu-binding": ",".join(str(int(num)) for num in bindings)} + + def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument, None]: + """Specify the real memory required per node + + This sets ``--memory-per-pe`` in megabytes + + :param memory_per_node: Per PE memory limit in megabytes + """ + return {"--memory-per-pe": str(memory_per_node)} + + def set_walltime(self, walltime: str) -> t.Union[StringArgument, None]: + """Set the walltime of the job + + Walltime is given in total number of seconds + + :param walltime: wall time + """ + return {"--cpu-time-limit": str(walltime)} + + def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + """Set the job to run in verbose mode + + This sets ``--debug`` arg to the highest level + + :param verbose: Whether the job should be run verbosely + """ + return {"--debug": 7} + + def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None],None]: + """Set the job to run in quiet mode + + This sets ``--quiet`` + + :param quiet: Whether the job should be run quietly + """ + return {"--quiet": None} + + def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: + """Format the environment variables for aprun + + :return: list of env vars + """ + formatted = [] + if env_vars: + for name, value in env_vars.items(): + formatted += ["-e", name + "=" + str(value)] + return formatted + + def format_launcher_args(self, launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + """Return a list of ALPS formatted run arguments + + :return: list of ALPS arguments for these settings + """ + # args launcher uses + args = [] + restricted = ["wdir"] + + for opt, value in launch_args.items(): + if opt not in restricted: + short_arg = bool(len(str(opt)) == 1) + prefix = "-" if short_arg else "--" + if not value: + args += [prefix + opt] + else: + if short_arg: + args += [prefix + opt, str(value)] + else: + args += ["=".join((prefix + opt, str(value)))] + return args \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/dragon.py b/smartsim/settingshold/translators/launch/dragon.py new file mode 100644 index 0000000000..9964e720e7 --- /dev/null +++ b/smartsim/settingshold/translators/launch/dragon.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +from ..launchArgTranslator import LaunchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...launchCommand import LauncherType +from smartsim.log import get_logger + +logger = get_logger(__name__) + +class DragonArgTranslator(LaunchArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.DragonLauncher.value + + def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: + """Set the number of nodes + + :param nodes: number of nodes to run with + """ + return {"nodes": nodes} + + def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + """Set the number of tasks for this job + + :param tasks_per_node: number of tasks per node + """ + return {"tasks-per-node": tasks_per_node} \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/local.py b/smartsim/settingshold/translators/launch/local.py new file mode 100644 index 0000000000..4c5a527114 --- /dev/null +++ b/smartsim/settingshold/translators/launch/local.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +from ..launchArgTranslator import LaunchArgTranslator +from ...launchCommand import LauncherType +from smartsim.log import get_logger +from ...common import IntegerArgument, StringArgument, FloatArgument + +logger = get_logger(__name__) + +class LocalArgTranslator(LaunchArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.LocalLauncher.value + + def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + """Build environment variable string + + :returns: formatted list of strings to export variables + """ + formatted = [] + for key, val in env_vars.items(): + if val is None: + formatted.append(f"{key}=") + else: + formatted.append(f"{key}={val}") + return formatted + + def format_launch_args(self, launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + """Build environment variable string + + :returns: formatted list of strings to export variables + """ + formatted = [] + for arg, value in launch_args.items(): + formatted.append(arg) + formatted.append(str(value)) + return formatted \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/lsf.py b/smartsim/settingshold/translators/launch/lsf.py new file mode 100644 index 0000000000..a51bc5f826 --- /dev/null +++ b/smartsim/settingshold/translators/launch/lsf.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +from ..launchArgTranslator import LaunchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...launchCommand import LauncherType +from smartsim.log import get_logger + +logger = get_logger(__name__) + +class JsrunArgTranslator(LaunchArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.LsfLauncher.value + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: + """Set the number of tasks for this job + + This sets ``--np`` + + :param tasks: number of tasks + """ + return {"np": int(tasks)} + + def set_binding(self, binding: str) -> t.Union[StringArgument, None]: + """Set binding + + This sets ``--bind`` + + :param binding: Binding, e.g. `packed:21` + """ + return {"bind": binding} + + def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + """Format environment variables. Each variable needs + to be passed with ``--env``. If a variable is set to ``None``, + its value is propagated from the current environment. + + :returns: formatted list of strings to export variables + """ + format_str = [] + for k, v in env_vars.items(): + if v: + format_str += ["-E", f"{k}={v}"] + else: + format_str += ["-E", f"{k}"] + return format_str + + def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + """Return a list of LSF formatted run arguments + + :return: list of LSF arguments for these settings + """ + # args launcher uses + args = [] + restricted = ["chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"] + for opt, value in launcher_args.items(): + if opt not in restricted: + short_arg = bool(len(str(opt)) == 1) + prefix = "-" if short_arg else "--" + if not value: + args += [prefix + opt] + else: + if short_arg: + args += [prefix + opt, str(value)] + else: + args += ["=".join((prefix + opt, str(value)))] + return args \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/mpi.py b/smartsim/settingshold/translators/launch/mpi.py new file mode 100644 index 0000000000..33e888621d --- /dev/null +++ b/smartsim/settingshold/translators/launch/mpi.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +from ..launchArgTranslator import LaunchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...launchCommand import LauncherType +from smartsim.log import get_logger + +logger = get_logger(__name__) + +class _BaseMPIArgTranslator(LaunchArgTranslator): + + def set_task_map(self, task_mapping: str) -> t.Union[StringArgument, None]: + """ Set ``mpirun`` task mapping + + this sets ``--map-by `` + + For examples, see the man page for ``mpirun`` + + :param task_mapping: task mapping + """ + return {"map-by": task_mapping} + + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: + """ Set the number of tasks for this job + + This sets ``--cpus-per-proc`` for MPI compliant implementations + + note: this option has been deprecated in openMPI 4.0+ + and will soon be replaced. + + :param cpus_per_task: number of tasks + """ + return {"cpus-per-proc": int(cpus_per_task)} + + def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument, None]: + """Copy the specified executable(s) to remote machines + + This sets ``--preload-binary`` + + :param dest_path: Destination path (Ignored) + """ + if dest_path is not None and isinstance(dest_path, str): + logger.warning( + ( + f"{type(self)} cannot set a destination path during broadcast. " + "Using session directory instead" + ) + ) + return {"preload-binary": dest_path} + + def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument, None]: + """ Specifies the cores to which MPI processes are bound + + This sets ``--bind-to`` for MPI compliant implementations + + :param bind_type: binding type + """ + return {"bind-to": bind_type} + + def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + """ Set the number of tasks per node + + :param tasks_per_node: number of tasks to launch per node + """ + return {"npernode": int(tasks_per_node)} + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: + """ Set the number of tasks for this job + + This sets ``-n`` for MPI compliant implementations + + :param tasks: number of tasks + """ + return {"n": int(tasks)} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + """ Set the hostlist for the ``mpirun`` command + + This sets ``--host`` + + :param host_list: list of host names + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"host": ",".join(host_list)} + + def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: + """ Use the contents of a file to set the hostlist + + This sets ``--hostfile`` + + :param file_path: Path to the hostlist file + """ + return {"hostfile": file_path} + + def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + """ Set the job to run in verbose mode + + This sets ``--verbose`` + + :param verbose: Whether the job should be run verbosely + """ + return {"verbose": None} + + def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None], None]: + """ Set the job to run in quiet mode + + This sets ``--quiet`` + + :param quiet: Whether the job should be run quietly + """ + return {"quiet": None} + + def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: + """ Format the environment variables for mpirun + + :return: list of env vars + """ + formatted = [] + env_string = "-x" + + if env_vars: + for name, value in env_vars.items(): + if value: + formatted += [env_string, "=".join((name, str(value)))] + else: + formatted += [env_string, name] + return formatted + +class MpiArgTranslator(_BaseMPIArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.MpirunLauncher.value + +class MpiexecArgTranslator(_BaseMPIArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.MpiexecLauncher.value + +class OrteArgTranslator(_BaseMPIArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.OrterunLauncher.value \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py new file mode 100644 index 0000000000..1063fdd417 --- /dev/null +++ b/smartsim/settingshold/translators/launch/pals.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +from ..launchArgTranslator import LaunchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...launchCommand import LauncherType +from smartsim.log import get_logger + +logger = get_logger(__name__) + +class PalsMpiexecArgTranslator(LaunchArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.PalsLauncher.value + + def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: + """ Specifies the cores to which MPI processes are bound + + This sets ``--bind-to`` for MPI compliant implementations + + :param bind_type: binding type + """ + return {"bind-to": bind_type} + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: + """ Set the number of tasks + + :param tasks: number of total tasks to launch + """ + return {"np": int(tasks)} + + def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument, None]: + """Copy the specified executable(s) to remote machines + + This sets ``--transfer`` + + :param dest_path: Destination path (Ignored) + """ + return {"transfer": str(dest_path)} + + def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + """ Set the number of tasks per node + + This sets ``--ppn`` + + :param tasks_per_node: number of tasks to launch per node + """ + return {"ppn": int(tasks_per_node)} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + """ Set the hostlist for the PALS ``mpiexec`` command + + This sets ``hosts`` + + :param host_list: list of host names + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"hosts": ",".join(host_list)} + + def format_launch_args(self, launcher_args: t.Dict[str, t.Union[str,int,float]]) -> t.Union[t.List[str],None]: + """ Return a list of MPI-standard formatted run arguments + + :return: list of MPI-standard arguments for these settings + """ + # args launcher uses + args = [] + restricted = ["wdir", "wd"] + + for opt, value in launcher_args.items(): + if opt not in restricted: + prefix = "--" + if not value: + args += [prefix + opt] + else: + args += [prefix + opt, str(value)] + + return args + + def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: + """ Format the environment variables for mpirun + + :return: list of env vars + """ + formatted = [] + + export_vars = [] + if env_vars: + for name, value in env_vars.items(): + if value: + formatted += ["--env", "=".join((name, str(value)))] + else: + export_vars.append(name) + + if export_vars: + formatted += ["--envlist", ",".join(export_vars)] + + return formatted \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py new file mode 100644 index 0000000000..f1aa21c0fc --- /dev/null +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from enum import Enum +import typing as t +import re +import os +from ..launchArgTranslator import LaunchArgTranslator +from ...common import IntegerArgument, StringArgument, FloatArgument +from ...launchCommand import LauncherType +from smartsim.log import get_logger + +logger = get_logger(__name__) + + +class SlurmArgTranslator(LaunchArgTranslator): + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.SlurmLauncher.value + + def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: + """ Set the number of nodes + + Effectively this is setting: ``srun --nodes `` + + :param nodes: nodes to launch on + :return: launcher argument + """ + return {"nodes": int(nodes)} + + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """ Specify the hostlist for this job + + This sets ``--nodelist`` + + :param host_list: hosts to launch on + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + elif not isinstance(host_list, list): + raise TypeError("host_list argument must be a string or list of strings") + elif not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return {"nodelist": ",".join(host_list)} + + def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: + """ Use the contents of a file to set the node list + + This sets ``--nodefile`` + + :param file_path: Path to the nodelist file + """ + return {"nodefile": file_path} + + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """ Specify a list of hosts to exclude for launching this job + + :param host_list: hosts to exclude + :raises TypeError: if not str or list of str + """ + if isinstance(host_list, str): + host_list = [host_list.strip()] + if not isinstance(host_list, list): + raise TypeError("host_list argument must be a list of strings") + if not all(isinstance(host, str) for host in host_list): + raise TypeError("host_list argument must be list of strings") + return { "exclude": ",".join(host_list)} + + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: + """ Set the number of cpus to use per task + + This sets ``--cpus-per-task`` + + :param num_cpus: number of cpus to use per task + """ + return {"cpus-per-task": int(cpus_per_task)} + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + """ Set the number of tasks for this job + + This sets ``--ntasks`` + + :param tasks: number of tasks + """ + return {"ntasks": int(tasks)} + + def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument,None]: + """ Set the number of tasks for this job + + This sets ``--ntasks-per-node`` + + :param tasks_per_node: number of tasks per node + """ + return {"ntasks-per-node": int(tasks_per_node)} + + def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> t.Union[StringArgument,None]: + """ Bind by setting CPU masks on tasks + + This sets ``--cpu-bind`` using the ``map_cpu:`` option + + :param bindings: List specifing the cores to which MPI processes are bound + """ + if isinstance(bindings, int): + bindings = [bindings] + return {"cpu_bind": "map_cpu:" + ",".join(str(num) for num in bindings)} + + def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument,None]: + """ Specify the real memory required per node + + This sets ``--mem`` in megabytes + + :param memory_per_node: Amount of memory per node in megabytes + """ + return {"mem": f"{memory_per_node}M"} + + def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument,None]: + """ Copy executable file to allocated compute nodes + + This sets ``--bcast`` + + :param dest_path: Path to copy an executable file + """ + return {"bcast": dest_path} + + def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """ Specify the node feature for this job + + This sets ``-C`` + + :param feature_list: node feature to launch on + :raises TypeError: if not str or list of str + """ + if isinstance(feature_list, str): + feature_list = [feature_list.strip()] + elif not all(isinstance(feature, str) for feature in feature_list): + raise TypeError("node_feature argument must be string or list of strings") + return {"C": ",".join(feature_list)} + + def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + """ Set the walltime of the job + + format = "HH:MM:SS" + + :param walltime: wall time + """ + pattern = r'^\d{2}:\d{2}:\d{2}$' + if walltime and re.match(pattern, walltime): + return {"time": str(walltime)} + else: + raise ValueError("Invalid walltime format. Please use 'HH:MM:SS' format.") + + def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + """ Set the job to run in verbose mode + + This sets ``--verbose`` + + :param verbose: Whether the job should be run verbosely + """ + return {"verbose": None} + + def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str, None], None]: + """Set the job to run in quiet mode + + This sets ``--quiet`` + + :param quiet: Whether the job should be run quietly + """ + return {"quiet": None} + + def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + """Return a list of slurm formatted launch arguments + + :return: list of slurm arguments for these settings + """ + # add additional slurm arguments based on key length + opts = [] + for opt, value in launcher_args.items(): + short_arg = bool(len(str(opt)) == 1) + prefix = "-" if short_arg else "--" + if not value: + opts += [prefix + opt] + else: + if short_arg: + opts += [prefix + opt, str(value)] + else: + opts += ["=".join((prefix + opt, str(value)))] + return opts + + def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + """Build bash compatible environment variable string for Slurm + + :returns: the formatted string of environment variables + """ + self._check_env_vars(env_vars) + return [f"{k}={v}" for k, v in env_vars.items() if "," not in str(v)] + + def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.Tuple[str, t.List[str]],None]: + """Build environment variable string for Slurm + + Slurm takes exports in comma separated lists + the list starts with all as to not disturb the rest of the environment + for more information on this, see the slurm documentation for srun + + :returns: the formatted string of environment variables + """ + self._check_env_vars(env_vars) + exportable_env, compound_env, key_only = [], [], [] + + for k, v in env_vars.items(): + kvp = f"{k}={v}" + + if "," in str(v): + key_only.append(k) + compound_env.append(kvp) + else: + exportable_env.append(kvp) + + # Append keys to exportable KVPs, e.g. `--export x1=v1,KO1,KO2` + fmt_exported_env = ",".join(v for v in exportable_env + key_only) + + return fmt_exported_env, compound_env + + def _check_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> None: + """Warn a user trying to set a variable which is set in the environment + + Given Slurm's env var precedence, trying to export a variable which is already + present in the environment will not work. + """ + for k, v in env_vars.items(): + if "," not in str(v): + # If a variable is defined, it will take precedence over --export + # we warn the user + preexisting_var = os.environ.get(k, None) + if preexisting_var is not None and preexisting_var != v: + msg = ( + f"Variable {k} is set to {preexisting_var} in current " + "environment. If the job is running in an interactive " + f"allocation, the value {v} will not be set. Please " + "consider removing the variable from the environment " + "and re-run the experiment." + ) + logger.warning(msg) \ No newline at end of file diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py new file mode 100644 index 0000000000..828f09165c --- /dev/null +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +import typing as t +# from launchcommand import LauncherType + +from smartsim.log import get_logger + +logger = get_logger(__name__) + +from ..common import IntegerArgument, StringArgument, FloatArgument + + +class LaunchArgTranslator(ABC): #ArgTranslator - LaunchArgTranslatro and BatchArgTranslator + """Abstract base class that defines all generic launcher + argument methods that are not supported. It is the + responsibility of child classes for each launcher to translate + the input parameter to a properly formatted launcher argument. + """ + + @abstractmethod + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + pass + + def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: + """ Convert the provide number of nodes into a properly formatted launcher + argument. + """ + logger.warning(f"set_nodes() not supported for {self.launcher_str()}.") + return None + + def set_hostlist(self, hostlist: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """ Convert the provide hostlist into a properly formatted launcher argument. + """ + logger.warning(f"set_hostlist() not supported for {self.launcher_str()}.") + return None + + def set_hostlist_from_file(self, hostlist: str) -> t.Union[StringArgument,None]: + """ Convert the file path into a properly formatted launcher argument. + """ + logger.warning(f"set_hostlist_from_file() not supported for {self.launcher_str()}.") + return None + + def set_excluded_hosts(self, hostlist: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """ Convert the hostlist into a properly formatted launcher argument. + """ + logger.warning(f"set_excluded_hosts() not supported for {self.launcher_str()}.") + return None + + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: + """ Convert the cpus_per_task into a properly formatted launcher argument. + """ + logger.warning(f"set_cpus_per_task() not supported for {self.launcher_str()}.") + return None + + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + """ Convert the tasks into a properly formatted launcher argument. + """ + logger.warning(f"set_tasks() not supported for {self.launcher_str()}.") + return None + + def set_tasks_per_node(self, tasks: int) -> t.Union[IntegerArgument,None]: + """ Convert the set_tasks_per_node into a properly formatted launcher argument. + """ + logger.warning(f"set_tasks_per_node() not supported for {self.launcher_str()}.") + return None + + def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> t.Union[StringArgument,None]: + """ Convert the cpu bindings into a properly formatted launcher argument. + """ + logger.warning(f"set_cpu_bindings() not supported for {self.launcher_str()}.") + return None + + def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument,None]: + """ Convert the real memory required per node into a properly formatted + launcher argument. + """ + logger.warning(f"set_memory_per_node() not supported for {self.launcher_str()}.") + return None + + def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument,None]: + """ Convert executable file to be copied to allocated compute nodes into + a properly formatted launcher argument. + """ + logger.warning(f"set_executable_broadcast() not supported for {self.launcher_str()}.") + return None + + def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + """ Convert node feature into a properly formatted launcher argument. + """ + logger.warning(f"set_node_feature() not supported for {self.launcher_str()}.") + return None + + def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + """ Convert walltime into a properly formatted launcher argument. + """ + logger.warning(f"set_walltime() not supported for {self.launcher_str()}.") + return None + + def set_binding(self, binding: str) -> t.Union[StringArgument,None]: + """Set binding + + This sets ``--bind`` + + :param binding: Binding, e.g. `packed:21` + """ + logger.warning(f"set_binding() not supported for {self.launcher_str()}.") + return None + + def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: + """Specifies the cores to which MPI processes are bound + + This sets ``--bind-to`` for MPI compliant implementations + + :param bind_type: binding type + """ + logger.warning(f"set_cpu_binding_type() not supported for {self.launcher_str()}.") + return None + + def set_task_map(self, task_mapping: str) -> t.Union[StringArgument,None]: + """Set ``mpirun`` task mapping + + this sets ``--map-by `` + + For examples, see the man page for ``mpirun`` + + :param task_mapping: task mapping + """ + logger.warning(f"set_task_map() not supported for {self.launcher_str()}.") + return None + + def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str, None], None]: + """Set the job to run in quiet mode + + This sets ``--quiet`` + + :param quiet: Whether the job should be run quietly + """ + logger.warning(f"set_quiet_launch() not supported for {self.launcher_str()}.") + return None + + def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + """Build bash compatible environment variable string for Slurm + + :returns: the formatted string of environment variables + """ + logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") + return None + + def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + """ Build formatted launch arguments + """ + logger.warning(f"format_launcher_args() not supported for {self.launcher_str()}.") + return None + + def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.Tuple[str, t.List[str]],None]: + """Build environment variable string for Slurm + + Slurm takes exports in comma separated lists + the list starts with all as to not disturb the rest of the environment + for more information on this, see the slurm documentation for srun + + :returns: the formatted string of environment variables + """ + logger.warning(f"format_comma_sep_env_vars() not supported for {self.launcher_str()}.") + return None + + def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + """Set the job to run in verbose mode + + This sets ``--verbose`` + + :param verbose: Whether the job should be run verbosely + """ + logger.warning(f"set_verbose_launch() not supported for {self.launcher_str()}.") + return None \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/settings_test/test_alpsLauncher.py new file mode 100644 index 0000000000..49a6980c0c --- /dev/null +++ b/tests/temp_tests/settings_test/test_alpsLauncher.py @@ -0,0 +1,94 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.alps import AprunArgTranslator +import pytest + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_cpus_per_task", (4,),4,"--cpus-per-pe",id="set_cpus_per_task"), + pytest.param("set_tasks", (4,),4,"--pes",id="set_tasks"), + pytest.param("set_tasks_per_node", (4,),4,"--pes-per-node",id="set_tasks_per_node"), + pytest.param("set_hostlist", ("host_A",),"host_A","--node-list",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","--node-list",id="set_hostlist_list[str]"), + pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","--node-list-file",id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("host_A",),"host_A","--exclude-node-list",id="set_excluded_hosts_str"), + pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","--exclude-node-list",id="set_excluded_hosts_list[str]"), + pytest.param("set_cpu_bindings", (4,),"4","--cpu-binding",id="set_cpu_bindings"), + pytest.param("set_cpu_bindings", ([4,4],),"4,4","--cpu-binding",id="set_cpu_bindings_list[str]"), + pytest.param("set_memory_per_node", (8000,),8000,"--memory-per-pe",id="set_memory_per_node"), + pytest.param("set_walltime", ("10:00:00",),"10:00:00","--cpu-time-limit",id="set_walltime"), + ], +) +def test_update_env_initialized(function, value, flag, result): + alpsLauncher = LaunchSettings(launcher="aprun") + assert alpsLauncher.launcher == "aprun" + assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) + getattr(alpsLauncher, function)(*value) + assert alpsLauncher.launcher_args[flag] == result + +def test_set_verbose_launch(): + slurmLauncher = LaunchSettings(launcher="aprun") + assert slurmLauncher.launcher == "aprun" + assert isinstance(slurmLauncher.arg_translator,AprunArgTranslator) + slurmLauncher.set_verbose_launch(True) + assert slurmLauncher.launcher_args == {'--debug': 7} + slurmLauncher.set_verbose_launch(False) + assert slurmLauncher.launcher_args == {} + +def test_set_quiet_launch(): + slurmLauncher = LaunchSettings(launcher="aprun") + assert slurmLauncher.launcher == "aprun" + assert isinstance(slurmLauncher.arg_translator,AprunArgTranslator) + slurmLauncher.set_quiet_launch(True) + assert slurmLauncher.launcher_args == {'--quiet': None} + slurmLauncher.set_quiet_launch(False) + assert slurmLauncher.launcher_args == {} + +def test_format_env_vars(): + env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} + slurmLauncher = LaunchSettings(launcher="aprun", env_vars=env_vars) + assert slurmLauncher.launcher == "aprun" + formatted = slurmLauncher.format_env_vars() + assert "OMP_NUM_THREADS=20" in formatted + assert "LOGGING=verbose" in formatted + + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_executable_broadcast"), + pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_task_map", ("task:map",), id="set_task_map"), + pytest.param("set_binding", ("bind",), id="set_binding"), + ], +) +def test_unimplimented_methods_throw_warning(caplog, method, params): + """Test methods not implemented throw warnings""" + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + slurmLauncher = LaunchSettings(launcher="slurm") + try: + getattr(slurmLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "slurm") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_aprunLauncher.py b/tests/temp_tests/settings_test/test_aprunLauncher.py new file mode 100644 index 0000000000..c385d09b7d --- /dev/null +++ b/tests/temp_tests/settings_test/test_aprunLauncher.py @@ -0,0 +1,138 @@ +import pytest +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.alps import AprunArgTranslator +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_cpus_per_task", (4,),4,"cpus-per-task",id="set_cpus_per_task"), + pytest.param("set_tasks", (4,),4,"pes",id="set_tasks"), + pytest.param("set_tasks_per_node", (4,),4,"pes-per-node",id="set_tasks_per_node"), + pytest.param("set_hostlist", ("host_A",),"host_A","node-list",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","node-list",id="set_hostlist_list[str]"), + pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","node-list-file",id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude-node-list",id="set_excluded_hosts_str"), + pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude-node-list",id="set_excluded_hosts_list[str]"), + pytest.param("set_cpu_bindings", (4,),"map_cpu:4","cpu-binding",id="set_cpu_bindings"), + pytest.param("set_cpu_bindings", ([4,4],),"map_cpu:4,4","cpu-binding",id="set_cpu_bindings_list[str]"), + pytest.param("set_memory_per_node", (8000,),"8000M","memory-per-pe",id="set_memory_per_node"), + pytest.param("set_walltime", ("10:00:00",),"10:00:00","cpu-time-limit",id="set_walltime"), + pytest.param("set_verbose_launch", (True,),None,"debug",id="set_verbose_launch"), + pytest.param("set_quiet_launch", (True,),None,"quiet",id="set_quiet_launch"), + ], +) +def test_update_env_initialized(function, value, flag, result): + aprunLauncher = LaunchSettings(launcher="aprun") + assert aprunLauncher.launcher == "aprun" + assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) + getattr(aprunLauncher, function)(*value) + assert aprunLauncher.launcher_args[flag] == result + +def test_set_verbose_launch(): + aprunLauncher = LaunchSettings(launcher="aprun") + assert aprunLauncher.launcher == "aprun" + assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) + aprunLauncher.set_verbose_launch(True) + assert aprunLauncher.launcher_args == {'verbose': None} + aprunLauncher.set_verbose_launch(False) + assert aprunLauncher.launcher_args == {} + +def test_set_quiet_launch(): + aprunLauncher = LaunchSettings(launcher="aprun") + assert aprunLauncher.launcher == "aprun" + assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) + aprunLauncher.set_quiet_launch(True) + assert aprunLauncher.launcher_args == {'quiet': None} + aprunLauncher.set_quiet_launch(False) + assert aprunLauncher.launcher_args == {} + +def test_format_env_vars(): + env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} + aprunLauncher = LaunchSettings(launcher="aprun", env_vars=env_vars) + assert aprunLauncher.launcher == "aprun" + aprunLauncher.update_env({"OMP_NUM_THREADS": 10}) + formatted = aprunLauncher.format_env_vars() + result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] + assert formatted == result + +def test_aprun_settings(): + aprunLauncher = LaunchSettings(launcher="aprun") + aprunLauncher.set_cpus_per_task(2) + aprunLauncher.set_tasks(100) + aprunLauncher.set_tasks_per_node(20) + formatted = aprunLauncher.format_run_args() + result = ["--cpus-per-pe=2", "--pes=100", "--pes-per-node=20"] + assert formatted == result + +def test_invalid_hostlist_format(): + """Test invalid hostlist formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(TypeError): + slurmLauncher.set_hostlist(["test",5]) + with pytest.raises(TypeError): + slurmLauncher.set_hostlist([5]) + with pytest.raises(TypeError): + slurmLauncher.set_hostlist(5) + +def test_invalid_exclude_hostlist_format(): + """Test invalid hostlist formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(TypeError): + slurmLauncher.set_excluded_hosts(["test",5]) + with pytest.raises(TypeError): + slurmLauncher.set_excluded_hosts([5]) + with pytest.raises(TypeError): + slurmLauncher.set_excluded_hosts(5) + +def test_invalid_walltime_format(): + """Test invalid walltime formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("11:11") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("ss:ss:ss") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("11:ss:ss") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("0s:ss:ss") + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_broadcast"), + pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_task_map", ("task:map",), id="set_task_map"), + pytest.param("set_binding", ("bind",), id="set_binding"), + ], +) +def test_unimplimented_methods_throw_warning(caplog, method, params): + """Test methods not implemented throw warnings""" + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + slurmLauncher = LaunchSettings(launcher="slurm") + try: + getattr(slurmLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "slurm") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/settings_test/test_dragonLauncher.py new file mode 100644 index 0000000000..ea9bf9aeec --- /dev/null +++ b/tests/temp_tests/settings_test/test_dragonLauncher.py @@ -0,0 +1,65 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.dragon import DragonArgTranslator +import pytest +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_tasks_per_node", (2,),2,"tasks-per-node",id="set_tasks_per_node"), + ], +) +def test_update_env_initialized(function, value, flag, result): + dragonLauncher = LaunchSettings(launcher="dragon") + assert dragonLauncher.launcher == "dragon" + assert isinstance(dragonLauncher.arg_translator,DragonArgTranslator) + getattr(dragonLauncher, function)(*value) + assert dragonLauncher.launcher_args[flag] == result + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_task_map", ("task:map",), id="set_task_map"), + pytest.param("set_hostlist", ("host_A",),id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),id="set_hostlist_list[str]"), + pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("host_A",),id="set_excluded_hosts_str"), + pytest.param("set_excluded_hosts", (["host_A","host_B"],),id="set_excluded_hosts_list[str]"), + pytest.param("set_cpu_bindings", (4,),id="set_cpu_bindings"), + pytest.param("set_cpu_bindings", ([4,4],),id="set_cpu_bindings_list[str]"), + pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), + pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), + pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), + pytest.param("set_walltime", ("10:00:00",),id="set_walltime"), + pytest.param("set_node_feature", ("P100",),id="set_node_feature"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + dragonLauncher = LaunchSettings(launcher="dragon") + try: + getattr(dragonLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "dragon") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_localLauncher.py b/tests/temp_tests/settings_test/test_localLauncher.py new file mode 100644 index 0000000000..3334481a6e --- /dev/null +++ b/tests/temp_tests/settings_test/test_localLauncher.py @@ -0,0 +1,157 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.local import LocalArgTranslator +import pytest +import logging + +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + localLauncher = LaunchSettings(launcher="local") + assert localLauncher.launcher_str() == "local" + +def test_launch_args_input_mutation(): + # Tests that the run args passed in are not modified after initialization + key0, key1, key2 = "arg0", "arg1", "arg2" + val0, val1, val2 = "val0", "val1", "val2" + + default_launcher_args = { + key0: val0, + key1: val1, + key2: val2, + } + localLauncher = LaunchSettings(launcher="local", launcher_args=default_launcher_args) + + # Confirm initial values are set + assert localLauncher.launcher_args[key0] == val0 + assert localLauncher.launcher_args[key1] == val1 + assert localLauncher.launcher_args[key2] == val2 + + # Update our common run arguments + val2_upd = f"not-{val2}" + default_launcher_args[key2] = val2_upd + + # Confirm previously created run settings are not changed + assert localLauncher.launcher_args[key2] == val2 + +@pytest.mark.parametrize( + "env_vars", + [ + pytest.param({}, id="no env vars"), + pytest.param({"env1": "abc"}, id="normal var"), + pytest.param({"env1": "abc,def"}, id="compound var"), + pytest.param({"env1": "xyz", "env2": "pqr"}, id="multiple env vars"), + ], +) +def test_update_env(env_vars): + """Ensure non-initialized env vars update correctly""" + localLauncher = LaunchSettings(launcher="local") + localLauncher.update_env(env_vars) + + assert len(localLauncher.env_vars) == len(env_vars.keys()) + +def test_format_run_args(): + localLauncher = LaunchSettings(launcher="local", launcher_args={"-np": 2}) + launch_args = localLauncher.format_launch_args() + assert type(launch_args) == type(list()) + assert launch_args == ["-np", "2"] + +@pytest.mark.parametrize( + "env_vars", + [ + pytest.param({"env1": None}, id="null value not allowed"), + pytest.param({"env1": {"abc"}}, id="set value not allowed"), + pytest.param({"env1": {"abc": "def"}}, id="dict value not allowed"), + ], +) +def test_update_env_null_valued(env_vars): + """Ensure validation of env var in update""" + orig_env = {} + + with pytest.raises(TypeError) as ex: + localLauncher = LaunchSettings(launcher="local", env_vars=orig_env) + localLauncher.update_env(env_vars) + +@pytest.mark.parametrize( + "env_vars", + [ + pytest.param({}, id="no env vars"), + pytest.param({"env1": "abc"}, id="normal var"), + pytest.param({"env1": "abc,def"}, id="compound var"), + pytest.param({"env1": "xyz", "env2": "pqr"}, id="multiple env vars"), + ], +) +def test_update_env_initialized(env_vars): + """Ensure update of initialized env vars does not overwrite""" + orig_env = {"key": "value"} + localLauncher = LaunchSettings(launcher="local", env_vars=orig_env) + localLauncher.update_env(env_vars) + + combined_keys = {k for k in env_vars.keys()} + combined_keys.update(k for k in orig_env.keys()) + + assert len(localLauncher.env_vars) == len(combined_keys) + assert {k for k in localLauncher.env_vars.keys()} == combined_keys + +def test_format_env_vars(): + env_vars={ + "A": "a", + "B": None, + "C": "", + "D": 12, + } + localLauncher = LaunchSettings(launcher="local", env_vars=env_vars) + assert localLauncher.launcher == "local" + assert isinstance(localLauncher.arg_translator, LocalArgTranslator) + assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_nodes", (2,), id="set_nodes"), + pytest.param("set_tasks", (2,), id="set_tasks"), + pytest.param("set_tasks_per_node", (3,), id="set_tasks_per_node"), + pytest.param("set_task_map", (3,), id="set_task_map"), + pytest.param("set_cpus_per_task", (4,), id="set_cpus_per_task"), + pytest.param("set_hostlist", ("hostlist",), id="set_hostlist"), + pytest.param("set_node_feature", ("P100",), id="set_node_feature"), + pytest.param( + "set_hostlist_from_file", ("~/hostfile",), id="set_hostlist_from_file" + ), + pytest.param("set_excluded_hosts", ("hostlist",), id="set_excluded_hosts"), + pytest.param("set_cpu_bindings", ([1, 2, 3],), id="set_cpu_bindings"), + pytest.param("set_memory_per_node", (16_000,), id="set_memory_per_node"), + pytest.param("set_verbose_launch", (False,), id="set_verbose_launch"), + pytest.param("set_quiet_launch", (True,), id="set_quiet_launch"), + pytest.param("set_broadcast", ("/tmp",), id="set_broadcast"), + pytest.param("set_walltime", ("00:55:00",), id="set_walltime"), + pytest.param("set_binding", ("packed:21",), id="set_binding"), + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + localLauncher = LaunchSettings(launcher="local") + try: + getattr(localLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "local") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_lsfLauncher.py b/tests/temp_tests/settings_test/test_lsfLauncher.py new file mode 100644 index 0000000000..d012ec5dd3 --- /dev/null +++ b/tests/temp_tests/settings_test/test_lsfLauncher.py @@ -0,0 +1,96 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.lsf import JsrunArgTranslator +import pytest +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_tasks", (2,),2,"np",id="set_tasks"), + pytest.param("set_binding", ("packed:21",),"packed:21","bind",id="set_binding"), + ], +) +def test_update_env_initialized(function, value, flag, result): + lsfLauncher = LaunchSettings(launcher="jsrun") + assert lsfLauncher.launcher == "jsrun" + assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) + getattr(lsfLauncher, function)(*value) + assert lsfLauncher.launcher_args[flag] == result + +def test_format_env_vars(): + env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} + lsfLauncher = LaunchSettings(launcher="jsrun", env_vars=env_vars) + assert lsfLauncher.launcher == "jsrun" + assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) + formatted = lsfLauncher.format_env_vars() + assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] + +def test_launch_args(): + """Test the possible user overrides through run_args""" + launch_args = { + "latency_priority": "gpu-gpu", + "immediate": None, + "d": "packed", # test single letter variables + "nrs": 10, + "np": 100, + } + lsfLauncher = LaunchSettings(launcher="jsrun", launcher_args=launch_args) + assert lsfLauncher.launcher == "jsrun" + formatted = lsfLauncher.format_launcher_args() + result = [ + "--latency_priority=gpu-gpu", + "--immediate", + "-d", + "packed", + "--nrs=10", + "--np=100", + ] + assert formatted == result + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_task_map", ("task:map",), id="set_task_map"), + pytest.param("set_nodes", (2,), id="set_nodes"), + pytest.param("set_hostlist", ("host_A",),id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),id="set_hostlist_list[str]"), + pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("host_A",),id="set_excluded_hosts_str"), + pytest.param("set_excluded_hosts", (["host_A","host_B"],),id="set_excluded_hosts_list[str]"), + pytest.param("set_cpu_bindings", (4,),id="set_cpu_bindings"), + pytest.param("set_cpu_bindings", ([4,4],),id="set_cpu_bindings_list[str]"), + pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), + pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), + pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), + pytest.param("set_walltime", ("10:00:00",),id="set_walltime"), + pytest.param("set_node_feature", ("P100",),id="set_node_feature"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + lsfLauncher = LaunchSettings(launcher="jsrun") + try: + getattr(lsfLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "jsrun") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_lsfScheduler.py b/tests/temp_tests/settings_test/test_lsfScheduler.py new file mode 100644 index 0000000000..26912e22d5 --- /dev/null +++ b/tests/temp_tests/settings_test/test_lsfScheduler.py @@ -0,0 +1,31 @@ +from smartsim.settingshold import BatchSettings +from smartsim.settingshold.translators.batch.lsf import BsubBatchArgTranslator +import pytest + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_nodes", (2,),2,"nnodes",id="set_nodes"), + pytest.param("set_walltime", ("10:00:00",),"10:00","W",id="set_walltime"), + pytest.param("set_hostlist", ("host_A",),""'"host_A"'"","m",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),""'"host_A host_B"'"","m",id="set_hostlist_list[str]"), + pytest.param("set_smts", (1,),1,"alloc_flags",id="set_smts"), + pytest.param("set_project", ("project",),"project","P",id="set_project"), + pytest.param("set_account", ("project",),"project","P",id="set_account"), + pytest.param("set_tasks", (2,),2,"n",id="set_tasks"), + pytest.param("set_queue", ("queue",),"queue","q",id="set_queue"), + ], +) +def test_update_env_initialized(function, value, flag, result): + lsfScheduler = BatchSettings(scheduler="jsrun") + getattr(lsfScheduler, function)(*value) + assert lsfScheduler.scheduler_args[flag] == result + +def test_create_bsub(): + batch_args = {"core_isolation": None} + lsfScheduler = BatchSettings(scheduler="jsrun", scheduler_args=batch_args) + lsfScheduler.set_nodes(1) + lsfScheduler.set_walltime("10:10:10") + lsfScheduler.set_queue("default") + args = lsfScheduler.format_batch_args() + assert args == ["-core_isolation", "-nnodes 1", "-W 10:10", "-q default"] \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/settings_test/test_mpiLauncher.py new file mode 100644 index 0000000000..6e7678f471 --- /dev/null +++ b/tests/temp_tests/settings_test/test_mpiLauncher.py @@ -0,0 +1,162 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator +import pytest +import logging +import itertools + +@pytest.mark.parametrize( + "l,function,value,result,flag", + [ + # Use OpenMPI style settigs for all launchers + *itertools.chain.from_iterable( + ( + ( + pytest.param(l, "set_task_map", ("taskmap",),"taskmap","map-by",id="set_task_map"), + pytest.param(l, "set_cpus_per_task", (2,),2,"cpus-per-proc",id="set_cpus_per_task"), + pytest.param(l, "set_cpu_binding_type", ("4",),"4","bind-to",id="set_cpu_binding_type"), + pytest.param(l, "set_tasks_per_node", (4,),4,"npernode",id="set_tasks_per_node"), + pytest.param(l, "set_tasks", (4,),4,"n",id="set_tasks"), + pytest.param(l, "set_executable_broadcast", ("broadcast",),"broadcast","preload-binary",id="set_executable_broadcast"), + pytest.param(l, "set_hostlist", ("host_A",),"host_A","host",id="set_hostlist_str"), + pytest.param(l, "set_hostlist", (["host_A","host_B"],),"host_A,host_B","host",id="set_hostlist_list[str]"), + pytest.param(l, "set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","hostfile",id="set_hostlist_from_file"), + pytest.param(l, "set_node_feature", ("P100",),id="set_node_feature"), + pytest.param(l, "set_binding", ("bind",), id="set_binding"), + ) + for l in ("mpirun", "orterun", "mpiexec") + )) + ], +) +def test_update_env_initialized(l,function, value, flag, result): + mpiSettings = LaunchSettings(launcher=l) + assert mpiSettings.launcher == l + getattr(mpiSettings, function)(*value) + assert mpiSettings.launcher_args[flag] == result + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param("mpirun", id="format_env"), + pytest.param("orterun", id="format_env"), + pytest.param("mpiexec", id="format_env"), + ], +) +def test_format_env(launcher): + env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} + mpiSettings = LaunchSettings(launcher=launcher, env_vars=env_vars) + assert mpiSettings.launcher == launcher + formatted = mpiSettings.format_env_vars() + result = [ + "-x", + "OMP_NUM_THREADS=20", + "-x", + "LOGGING=verbose", + ] + assert formatted == result + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param("mpirun", id="format_env"), + pytest.param("orterun", id="format_env"), + pytest.param("mpiexec", id="format_env"), + ], +) +def test_set_verbose_launch(launcher): + mpiSettings = LaunchSettings(launcher=launcher) + assert mpiSettings.launcher == launcher + mpiSettings.set_verbose_launch(True) + assert mpiSettings.launcher_args == {'verbose': None} + mpiSettings.set_verbose_launch(False) + assert mpiSettings.launcher_args == {} + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param("mpirun", id="format_env"), + pytest.param("orterun", id="format_env"), + pytest.param("mpiexec", id="format_env"), + ], +) +def test_set_quiet_launch(launcher): + mpiSettings = LaunchSettings(launcher=launcher) + assert mpiSettings.launcher == launcher + mpiSettings.set_quiet_launch(True) + assert mpiSettings.launcher_args == {'quiet': None} + mpiSettings.set_quiet_launch(False) + assert mpiSettings.launcher_args == {} + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param("mpirun", id="format_env"), + pytest.param("orterun", id="format_env"), + pytest.param("mpiexec", id="format_env"), + ], +) +def test_invalid_hostlist_format(launcher): + """Test invalid hostlist formats""" + mpiSettings = LaunchSettings(launcher=launcher) + with pytest.raises(TypeError): + mpiSettings.set_hostlist(["test",5]) + with pytest.raises(TypeError): + mpiSettings.set_hostlist([5]) + with pytest.raises(TypeError): + mpiSettings.set_hostlist(5) + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param("mpirun", id="launcher_str_mpirun"), + pytest.param("orterun", id="launcher_str_orterun"), + pytest.param("mpiexec", id="launcher_str_mpiexec"), + ], +) +def test_launcher_str(launcher): + mpiLauncher = LaunchSettings(launcher=launcher) + assert mpiLauncher.launcher_str() == launcher + +@pytest.mark.parametrize( + "l,method,params", + [ + # Use OpenMPI style settigs for all launchers + *itertools.chain.from_iterable( + ( + ( + pytest.param(l, "set_nodes", (1,), id="set_nodes"), + pytest.param(l, "set_excluded_hosts", ("hosts",), id="set_excluded_hosts"), + pytest.param(l, "set_cpu_bindings", (1,), id="set_cpu_bindings"), + pytest.param(l, "set_memory_per_node", (3000,), id="set_memory_per_node"), + pytest.param(l, "set_binding", ("bind",), id="set_binding"), + ) + for l in ("mpirun", "orterun", "mpiexec") + )) + ], +) +def test_unimplimented_setters_throw_warning(l, caplog, method, params): + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + mpiSettings = LaunchSettings(launcher=l) + try: + getattr(mpiSettings, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and l) in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/settings_test/test_palsLauncher.py new file mode 100644 index 0000000000..3a01e0633c --- /dev/null +++ b/tests/temp_tests/settings_test/test_palsLauncher.py @@ -0,0 +1,79 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.pals import PalsMpiexecArgTranslator +import pytest +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_cpu_binding_type", ("bind",),"bind","bind-to",id="set_cpu_binding_type"), + pytest.param("set_tasks", (2,),2,"np",id="set_tasks"), + pytest.param("set_tasks_per_node", (2,),2,"ppn",id="set_tasks_per_node"), + pytest.param("set_hostlist", ("host_A",),"host_A","hosts",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hosts",id="set_hostlist_list[str]"), + pytest.param("set_executable_broadcast", ("broadcast",),"broadcast","transfer",id="set_executable_broadcast"), + ], +) +def test_update_env_initialized(function, value, flag, result): + palsLauncher = LaunchSettings(launcher="pals") + getattr(palsLauncher, function)(*value) + assert palsLauncher.launcher == "pals" + assert isinstance(palsLauncher.arg_translator,PalsMpiexecArgTranslator) + assert palsLauncher.launcher_args[flag] == result + assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] + +def test_format_env_vars(): + env_vars = {"FOO_VERSION": "3.14", "PATH": None, "LD_LIBRARY_PATH": None} + palsLauncher = LaunchSettings(launcher="pals", env_vars=env_vars) + formatted = " ".join(palsLauncher.format_env_vars()) + expected = "--env FOO_VERSION=3.14 --envlist PATH,LD_LIBRARY_PATH" + assert formatted == expected + +def test_invalid_hostlist_format(): + """Test invalid hostlist formats""" + palsLauncher = LaunchSettings(launcher="pals") + with pytest.raises(TypeError): + palsLauncher.set_hostlist(["test",5]) + with pytest.raises(TypeError): + palsLauncher.set_hostlist([5]) + with pytest.raises(TypeError): + palsLauncher.set_hostlist(5) + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_task_map", ("task:map",), id="set_task_map"), + pytest.param("set_cpus_per_task", ("task:map",), id="set_cpus_per_task"), + pytest.param("set_quiet_launch", ("task:map",), id="set_quiet_launch"), + pytest.param("set_walltime", ("task:map",), id="set_walltime"), + pytest.param("set_node_feature", ("P100",),id="set_node_feature"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + palsLauncher = LaunchSettings(launcher="pals") + try: + getattr(palsLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "pals") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_pbsScheduler.py b/tests/temp_tests/settings_test/test_pbsScheduler.py new file mode 100644 index 0000000000..8075255a09 --- /dev/null +++ b/tests/temp_tests/settings_test/test_pbsScheduler.py @@ -0,0 +1,37 @@ +from smartsim.settingshold import BatchSettings +from smartsim.settingshold.translators.batch.pbs import QsubBatchArgTranslator +import pytest +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_walltime", ("10:00:00",),"10:00:00","walltime",id="set_walltime"), + pytest.param("set_account", ("account",),"account","A",id="set_account"), + pytest.param("set_queue", ("queue",),"queue","q",id="set_queue"), + pytest.param("set_ncpus", (2,),2,"ppn",id="set_ncpus"), + pytest.param("set_hostlist", ("host_A",),"host_A","hostname",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hostname",id="set_hostlist_list[str]"), + ], +) +def test_update_env_initialized(function, value, flag, result): + pbsScheduler = BatchSettings(scheduler="qsub") + getattr(pbsScheduler, function)(*value) + assert pbsScheduler.scheduler_args[flag] == result + +def test_create_pbs_batch(): + pbsScheduler = BatchSettings(scheduler="qsub") + pbsScheduler.set_nodes(1) + pbsScheduler.set_walltime("10:00:00") + pbsScheduler.set_queue("default") + pbsScheduler.set_account("myproject") + pbsScheduler.set_ncpus(10) + args = pbsScheduler.format_batch_args() + print(f"here: {args}") + assert args == [ + "-l nodes=1:ncpus=10", + "-l walltime=10:00:00", + "-q default", + "-A myproject", + ] \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py new file mode 100644 index 0000000000..e72bf38bac --- /dev/null +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -0,0 +1,187 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.translators.launch.slurm import SlurmArgTranslator +import pytest +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_hostlist", ("host_A",),"host_A","nodelist",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","nodelist",id="set_hostlist_list[str]"), + pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","nodefile",id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude",id="set_excluded_hosts_str"), + pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude",id="set_excluded_hosts_list[str]"), + pytest.param("set_cpus_per_task", (4,),4,"cpus-per-task",id="set_cpus_per_task"), + pytest.param("set_tasks", (4,),4,"ntasks",id="set_tasks"), + pytest.param("set_tasks_per_node", (4,),4,"ntasks-per-node",id="set_tasks_per_node"), + pytest.param("set_cpu_bindings", (4,),"map_cpu:4","cpu_bind",id="set_cpu_bindings"), + pytest.param("set_cpu_bindings", ([4,4],),"map_cpu:4,4","cpu_bind",id="set_cpu_bindings_list[str]"), + pytest.param("set_memory_per_node", (8000,),"8000M","mem",id="set_memory_per_node"), + pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_broadcast"), + pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), + pytest.param("set_walltime", ("10:00:00",),"10:00:00","time",id="set_walltime"), + pytest.param("set_verbose_launch", (True,),None,"verbose",id="set_walltime"), + ], +) +def test_update_env_initialized(function, value, flag, result): + slurmLauncher = LaunchSettings(launcher="slurm") + assert slurmLauncher.launcher == "slurm" + assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) + getattr(slurmLauncher, function)(*value) + assert slurmLauncher.launcher_args[flag] == result + +def test_set_verbose_launch(): + slurmLauncher = LaunchSettings(launcher="slurm") + assert slurmLauncher.launcher == "slurm" + assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) + slurmLauncher.set_verbose_launch(True) + assert slurmLauncher.launcher_args == {'verbose': None} + slurmLauncher.set_verbose_launch(False) + assert slurmLauncher.launcher_args == {} + +def test_set_quiet_launch(): + slurmLauncher = LaunchSettings(launcher="slurm") + assert slurmLauncher.launcher == "slurm" + assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) + slurmLauncher.set_quiet_launch(True) + assert slurmLauncher.launcher_args == {'quiet': None} + slurmLauncher.set_quiet_launch(False) + assert slurmLauncher.launcher_args == {} + +def test_format_env_vars(): + """Test format_env_vars runs correctly""" + env_vars={ + "OMP_NUM_THREADS": 20, + "LOGGING": "verbose", + "SSKEYIN": "name_0,name_1", + } + slurmLauncher = LaunchSettings(launcher="slurm", env_vars=env_vars) + assert slurmLauncher.launcher == "slurm" + assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) + formatted = slurmLauncher.format_env_vars() + assert "OMP_NUM_THREADS=20" in formatted + assert "LOGGING=verbose" in formatted + assert all("SSKEYIN" not in x for x in formatted) + +def test_format_comma_sep_env_vars(): + """Test format_comma_sep_env_vars runs correctly""" + env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} + slurmLauncher = LaunchSettings(launcher="slurm", env_vars=env_vars) + formatted, comma_separated_formatted = slurmLauncher.format_comma_sep_env_vars() + assert "OMP_NUM_THREADS" in formatted + assert "LOGGING" in formatted + assert "SSKEYIN" in formatted + assert "name_0,name_1" not in formatted + assert "SSKEYIN=name_0,name_1" in comma_separated_formatted + +def test_srun_settings(): + """Test format_launcher_args runs correctly""" + slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher.set_nodes(5) + slurmLauncher.set_cpus_per_task(2) + slurmLauncher.set_tasks(100) + slurmLauncher.set_tasks_per_node(20) + formatted = slurmLauncher.format_launcher_args() + result = ["--nodes=5", "--cpus-per-task=2", "--ntasks=100", "--ntasks-per-node=20"] + assert formatted == result + +def test_srun_launcher_args(): + """Test the possible user overrides through run_args""" + launcher_args = { + "account": "A3123", + "exclusive": None, + "C": "P100", # test single letter variables + "nodes": 10, + "ntasks": 100, + } + slurmLauncher = LaunchSettings(launcher="slurm", launcher_args=launcher_args) + formatted = slurmLauncher.format_launcher_args() + result = [ + "--account=A3123", + "--exclusive", + "-C", + "P100", + "--nodes=10", + "--ntasks=100", + ] + assert formatted == result + +def test_invalid_hostlist_format(): + """Test invalid hostlist formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(TypeError): + slurmLauncher.set_hostlist(["test",5]) + with pytest.raises(TypeError): + slurmLauncher.set_hostlist([5]) + with pytest.raises(TypeError): + slurmLauncher.set_hostlist(5) + +def test_invalid_exclude_hostlist_format(): + """Test invalid hostlist formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(TypeError): + slurmLauncher.set_excluded_hosts(["test",5]) + with pytest.raises(TypeError): + slurmLauncher.set_excluded_hosts([5]) + with pytest.raises(TypeError): + slurmLauncher.set_excluded_hosts(5) + +def test_invalid_node_feature_format(): + """Test invalid node feature formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(TypeError): + slurmLauncher.set_node_feature(["test",5]) + with pytest.raises(TypeError): + slurmLauncher.set_node_feature([5]) + with pytest.raises(TypeError): + slurmLauncher.set_node_feature(5) + +def test_invalid_walltime_format(): + """Test invalid walltime formats""" + slurmLauncher = LaunchSettings(launcher="slurm") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("11:11") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("ss:ss:ss") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("11:ss:ss") + with pytest.raises(ValueError): + slurmLauncher.set_walltime("0s:ss:ss") + +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_task_map", ("task:map",), id="set_task_map"), + pytest.param("set_binding", ("bind",), id="set_binding"), + ], +) +def test_unimplimented_methods_throw_warning(caplog, method, params): + """Test methods not implemented throw warnings""" + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + slurmLauncher = LaunchSettings(launcher="slurm") + try: + getattr(slurmLauncher, method)(*params) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and ("not supported" and "slurm") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_slurmScheduler.py b/tests/temp_tests/settings_test/test_slurmScheduler.py new file mode 100644 index 0000000000..021ac6efd0 --- /dev/null +++ b/tests/temp_tests/settings_test/test_slurmScheduler.py @@ -0,0 +1,109 @@ +from smartsim.settingshold import BatchSettings +from smartsim.settingshold.translators.batch.slurm import SlurmBatchArgTranslator +import pytest +import logging + +@pytest.mark.parametrize( + "function,value,result,flag", + [ + pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_walltime", ("10:00:00",),"10:00:00","time",id="set_walltime"), + pytest.param("set_account", ("account",),"account","account",id="set_account"), + pytest.param("set_partition", ("partition",),"partition","partition",id="set_partition"), + pytest.param("set_queue", ("partition",),"partition","partition",id="set_queue"), + pytest.param("set_cpus_per_task", (2,),2,"cpus-per-task",id="set_cpus_per_task"), + pytest.param("set_hostlist", ("host_A",),"host_A","nodelist",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","nodelist",id="set_hostlist_list[str]"), + ], +) +def test_update_env_initialized(function, value, flag, result): + slurmScheduler = BatchSettings(scheduler="slurm") + getattr(slurmScheduler, function)(*value) + assert slurmScheduler.scheduler_args[flag] == result + +def test_create_sbatch(): + batch_args = {"exclusive": None, "oversubscribe": None} + slurmScheduler = BatchSettings(scheduler="slurm", scheduler_args=batch_args) + assert isinstance(slurmScheduler.arg_translator, SlurmBatchArgTranslator) + #assert slurmScheduler.batch_args["partition"] == "default" + args = slurmScheduler.format_batch_args() + assert args == [ + "--exclusive", + "--oversubscribe" + ] + +def test_launch_args_input_mutation(): + # Tests that the run args passed in are not modified after initialization + key0, key1, key2 = "arg0", "arg1", "arg2" + val0, val1, val2 = "val0", "val1", "val2" + + default_scheduler_args = { + key0: val0, + key1: val1, + key2: val2, + } + slurmScheduler = BatchSettings(scheduler="slurm", scheduler_args=default_scheduler_args) + + # Confirm initial values are set + assert slurmScheduler.scheduler_args[key0] == val0 + assert slurmScheduler.scheduler_args[key1] == val1 + assert slurmScheduler.scheduler_args[key2] == val2 + + # Update our common run arguments + val2_upd = f"not-{val2}" + default_scheduler_args[key2] = val2_upd + + # Confirm previously created run settings are not changed + assert slurmScheduler.scheduler_args[key2] == val2 + +def test_sbatch_settings(): + scheduler_args = {"nodes": 1, "time": "10:00:00", "account": "A3123"} + sbatch = BatchSettings(scheduler="slurm",scheduler_args=scheduler_args) + formatted = sbatch.format_batch_args() + result = ["--nodes=1", "--time=10:00:00", "--account=A3123"] + assert formatted == result + + +def test_sbatch_manual(): + sbatch = BatchSettings(scheduler="slurm") + sbatch.set_nodes(5) + sbatch.set_account("A3531") + sbatch.set_walltime("10:00:00") + formatted = sbatch.format_batch_args() + result = ["--nodes=5", "--account=A3531", "--time=10:00:00"] + assert formatted == result + +# @pytest.mark.parametrize( +# "method,params", +# [ +# pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), +# pytest.param("set_task_map", ("task:map",), id="set_task_map"), +# ], +# ) +# def test_unimplimented_setters_throw_warning(caplog, method, params): +# from smartsim.settings.base import logger + +# prev_prop = logger.propagate +# logger.propagate = True + +# with caplog.at_level(logging.WARNING): +# caplog.clear() +# launcher = BatchSettings(scheduler="slurm") +# try: +# getattr(launcher, method)(*params) +# finally: +# logger.propagate = prev_prop + +# for rec in caplog.records: +# if ( +# logging.WARNING <= rec.levelno < logging.ERROR +# and ("not supported" and "slurm") in rec.msg +# ): +# break +# else: +# pytest.fail( +# ( +# f"No message stating method `{method}` is not " +# "implemented at `warning` level" +# ) +# ) \ No newline at end of file diff --git a/tests/temp_tests/settings_tests.py b/tests/temp_tests/settings_tests.py deleted file mode 100644 index 7ba055f974..0000000000 --- a/tests/temp_tests/settings_tests.py +++ /dev/null @@ -1,123 +0,0 @@ -from smartsim.settings import RunSettings, SrunSettings, PalsMpiexecSettings, MpirunSettings, MpiexecSettings, OrterunSettings, JsrunSettings, AprunSettings, BsubBatchSettings, QsubBatchSettings, SbatchSettings -import os -from shutil import which -import pytest -import itertools -import os.path as osp - -env_vars = {"k1": "v1", "k2": "v2"} -run_args = {"envlist": "SPAM"} - -# Test that mpi RunSetting classes create without error -@pytest.mark.parametrize( - "settings_type, env_vars, run_args", - [ - pytest.param( - MpirunSettings, - env_vars, - run_args, - id=f"mpirun", - ), - pytest.param( - OrterunSettings, - env_vars, - run_args, - id=f"orterun", - ) - ] -) -def test_mpi_instantiate_run_settings( - settings_type, env_vars, run_args -): - settings = settings_type(run_args=run_args, env_vars=env_vars, fail_if_missing_exec=False) - assert settings.env_vars == env_vars - assert settings.run_args == run_args - assert isinstance(settings, settings_type) - -# Test that RunSetting classes create without error -@pytest.mark.parametrize( - "settings_type, env_vars, run_args", - [ - pytest.param( - SrunSettings, - env_vars, - run_args, - id=f"srun", - ), - pytest.param( - PalsMpiexecSettings, - env_vars, - run_args, - id=f"mpiexec", - ), - pytest.param( - JsrunSettings, - env_vars, - run_args, - id="jsrun", - ), - pytest.param( - RunSettings, - env_vars, - run_args, - id="local", - ), - pytest.param( - AprunSettings, - env_vars, - run_args, - id="aprun", - ) - ] -) -def test_instantiate_run_settings( - settings_type, env_vars, run_args -): - settings = settings_type(run_args=run_args, env_vars=env_vars) - assert settings.env_vars == env_vars - assert settings.run_args == run_args - assert isinstance(settings, settings_type) - -nodes = 4 -time = "10:00:00" -account = "1234" - -# Test that BatchSettings classes create without error -# This currently does not work, need to unify how we treat each settings class -@pytest.mark.parametrize( - "settings_type, nodes, node_flag, time, account", - [ - pytest.param( - BsubBatchSettings, - nodes, - "nnodes", - time, - account, - id=f"bsub", - ), - pytest.param( - QsubBatchSettings, - nodes, - "nodes", - time, - account, - id="qsub", - ), - pytest.param( - SbatchSettings, - nodes, - "nodes", - time, - account, - id="sbatch", - ) - ] -) -def test_instantiate_batch_settings( - settings_type, nodes, node_flag, time, account -): - batch_settings = settings_type(nodes=nodes, time=time, account=account) - assert batch_settings.resources[node_flag] == nodes - assert batch_settings.batch_args["time"] == time - assert batch_settings.batch_args["account"] == account - assert isinstance(batch_settings, settings_type) \ No newline at end of file From a2a3d31500fee9cd50ed168e7995ceaed43bd48c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 15:46:34 -0500 Subject: [PATCH 02/43] updating --- smartsim/settingshold/batchCommand.py | 3 + smartsim/settingshold/launchCommand.py | 3 + smartsim/settingshold/launchSettings.py | 56 ++++++++------ .../translators/batchArgTranslator.py | 4 +- .../settingshold/translators/launch/local.py | 13 ++-- .../settingshold/translators/launch/mpi.py | 30 +++++++- .../translators/launchArgTranslator.py | 2 +- .../settings_test/test_localLauncher.py | 32 ++++---- .../settings_test/test_mpiLauncher.py | 75 ++++++++++++------- 9 files changed, 140 insertions(+), 78 deletions(-) diff --git a/smartsim/settingshold/batchCommand.py b/smartsim/settingshold/batchCommand.py index b9c43a6acd..09c63111d8 100644 --- a/smartsim/settingshold/batchCommand.py +++ b/smartsim/settingshold/batchCommand.py @@ -1,6 +1,9 @@ from enum import Enum class SchedulerType(Enum): + """ Schedulers that are supported by + SmartSim. + """ SlurmLauncher = "sbatch" PbsLauncher = "qsub" LsfLauncher = "bsub" \ No newline at end of file diff --git a/smartsim/settingshold/launchCommand.py b/smartsim/settingshold/launchCommand.py index fd67814673..a3e346e9ba 100644 --- a/smartsim/settingshold/launchCommand.py +++ b/smartsim/settingshold/launchCommand.py @@ -1,6 +1,9 @@ from enum import Enum class LauncherType(Enum): + """ Launchers that are supported by + SmartSim. + """ DragonLauncher = "dragon" SlurmLauncher = "slurm" PalsLauncher = "pals" diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index 3ed744b9b3..470468a7e4 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -5,7 +5,7 @@ from smartsim.log import get_logger - +from .._core.utils.helpers import fmt_dict from .launchCommand import LauncherType from .translators.launch.alps import AprunArgTranslator from .translators.launch.lsf import JsrunArgTranslator @@ -20,20 +20,6 @@ logger = get_logger(__name__) -class SupportedLaunchers(Enum): - """ Launchers that are supported by - SmartSim. - """ - local = "local" - dragon = "dragon" - slurm = "slurm" - mpiexec = "mpiexec" - mpirun = "mpirun" - orterun = "orterun" - aprun = "aprun" - jsrun = "jsrun" - pals = "pals" - class LaunchSettings(): def __init__( self, @@ -53,22 +39,22 @@ def __init__( 'dragon': DragonArgTranslator(), 'local': LocalArgTranslator(), } - if launcher in launcher_to_translator: + if launcher.value in launcher_to_translator: self.launcher = launcher else: raise ValueError(f"'{launcher}' is not a valid launcher name.") - process_env_vars(env_vars) + #process_env_vars(env_vars) self.env_vars = env_vars or {} # TODO check and preporcess launcher_args self.launcher_args = launcher_args or {} - self.arg_translator = t.cast(LaunchArgTranslator,launcher_to_translator.get(launcher)) + self.arg_translator = t.cast(LaunchArgTranslator,launcher_to_translator.get(launcher.value)) @property def launcher_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: - """Return an immutable list of attached run arguments. + """Return an immutable list of attached launcher arguments. :returns: attached run arguments """ @@ -76,12 +62,28 @@ def launcher_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: @launcher_args.setter def launcher_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> None: - """Set the run arguments. + """Set the launcher arguments. :param value: run arguments """ self._launcher_args = copy.deepcopy(value) + @property + def env_vars(self) -> t.Dict[str, t.Optional[str]]: + """Return an immutable list of attached environment variables. + + :returns: attached environment variables + """ + return self._env_vars + + @env_vars.setter + def env_vars(self, value: t.Dict[str, t.Optional[str]]) -> None: + """Set the environment variables. + + :param value: environment variables + """ + self._env_vars = copy.deepcopy(value) + def launcher_str(self) -> str: """ Get the string representation of the launcher """ @@ -340,10 +342,20 @@ def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: raise TypeError("Argument name should be of type str") # if value is not None and not isinstance(value, str): # raise TypeError("Argument value should be of type str or None") - # arg = arg.strip().lstrip("-") + #arg = arg.strip().lstrip("-") # if not condition: # logger.info(f"Could not set argument '{arg}': condition not met") # return if key in self.launcher_args and arg != self.launcher_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{arg}'") - self.launcher_args[key] = arg \ No newline at end of file + self.launcher_args[key] = arg + + def __str__(self) -> str: # pragma: no-cover + string = "" + if self.launcher: + string += f"\nLauncher: {self.launcher}" + if self.launcher_args: + string += f"\Launch Arguments:\n{fmt_dict(self.launcher_args)}" + if self.env_vars: + string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" + return string \ No newline at end of file diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py index 1dbea70712..d9945a6c05 100644 --- a/smartsim/settingshold/translators/batchArgTranslator.py +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -11,8 +11,8 @@ from ..common import IntegerArgument, StringArgument, FloatArgument -class BatchArgTranslator(ABC): #ArgTranslator - LaunchArgTranslatro and BatchArgTranslator - """Abstract base class that defines all generic launcher +class BatchArgTranslator(ABC): + """Abstract base class that defines all generic scheduler argument methods that are not supported. It is the responsibility of child classes for each launcher to translate the input parameter to a properly formatted launcher argument. diff --git a/smartsim/settingshold/translators/launch/local.py b/smartsim/settingshold/translators/launch/local.py index 4c5a527114..2a24540a07 100644 --- a/smartsim/settingshold/translators/launch/local.py +++ b/smartsim/settingshold/translators/launch/local.py @@ -1,11 +1,10 @@ from __future__ import annotations -from enum import Enum import typing as t from ..launchArgTranslator import LaunchArgTranslator from ...launchCommand import LauncherType from smartsim.log import get_logger -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import StringArgument logger = get_logger(__name__) @@ -16,7 +15,7 @@ def launcher_str(self) -> str: """ return LauncherType.LocalLauncher.value - def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: """Build environment variable string :returns: formatted list of strings to export variables @@ -29,13 +28,13 @@ def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.L formatted.append(f"{key}={val}") return formatted - def format_launch_args(self, launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: - """Build environment variable string + def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + """Build launcher argument string - :returns: formatted list of strings to export variables + :returns: formatted list of launcher arguments """ formatted = [] - for arg, value in launch_args.items(): + for arg, value in launcher_args.items(): formatted.append(arg) formatted.append(str(value)) return formatted \ No newline at end of file diff --git a/smartsim/settingshold/translators/launch/mpi.py b/smartsim/settingshold/translators/launch/mpi.py index 33e888621d..0bdf8551c8 100644 --- a/smartsim/settingshold/translators/launch/mpi.py +++ b/smartsim/settingshold/translators/launch/mpi.py @@ -1,9 +1,8 @@ from __future__ import annotations -from enum import Enum import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from ...launchCommand import LauncherType from smartsim.log import get_logger @@ -109,6 +108,15 @@ def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict """ return {"verbose": None} + def set_walltime(self, walltime: str) -> None: + """Set the maximum number of seconds that a job will run + + This sets ``--timeout`` + + :param walltime: number like string of seconds that a job will run in secs + """ + return {"timeout": walltime} + def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None], None]: """ Set the job to run in quiet mode @@ -134,6 +142,24 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> formatted += [env_string, name] return formatted + def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + """Return a list of MPI-standard formatted run arguments + + :return: list of MPI-standard arguments for these settings + """ + # args launcher uses + args = [] + restricted = ["wdir", "wd"] + + for opt, value in launcher_args.items(): + if opt not in restricted: + prefix = "--" + if not value: + args += [prefix + opt] + else: + args += [prefix + opt, str(value)] + return args + class MpiArgTranslator(_BaseMPIArgTranslator): def launcher_str(self) -> str: diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py index 828f09165c..20f8f27ba4 100644 --- a/smartsim/settingshold/translators/launchArgTranslator.py +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -11,7 +11,7 @@ from ..common import IntegerArgument, StringArgument, FloatArgument -class LaunchArgTranslator(ABC): #ArgTranslator - LaunchArgTranslatro and BatchArgTranslator +class LaunchArgTranslator(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate diff --git a/tests/temp_tests/settings_test/test_localLauncher.py b/tests/temp_tests/settings_test/test_localLauncher.py index 3334481a6e..da9eb37af5 100644 --- a/tests/temp_tests/settings_test/test_localLauncher.py +++ b/tests/temp_tests/settings_test/test_localLauncher.py @@ -1,12 +1,13 @@ from smartsim.settingshold import LaunchSettings from smartsim.settingshold.translators.launch.local import LocalArgTranslator +from smartsim.settingshold.launchCommand import LauncherType import pytest import logging def test_launcher_str(): """Ensure launcher_str returns appropriate value""" - localLauncher = LaunchSettings(launcher="local") - assert localLauncher.launcher_str() == "local" + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) + assert localLauncher.launcher_str() == LauncherType.LocalLauncher.value def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization @@ -18,7 +19,7 @@ def test_launch_args_input_mutation(): key1: val1, key2: val2, } - localLauncher = LaunchSettings(launcher="local", launcher_args=default_launcher_args) + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args=default_launcher_args) # Confirm initial values are set assert localLauncher.launcher_args[key0] == val0 @@ -43,16 +44,15 @@ def test_launch_args_input_mutation(): ) def test_update_env(env_vars): """Ensure non-initialized env vars update correctly""" - localLauncher = LaunchSettings(launcher="local") + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) localLauncher.update_env(env_vars) assert len(localLauncher.env_vars) == len(env_vars.keys()) -def test_format_run_args(): - localLauncher = LaunchSettings(launcher="local", launcher_args={"-np": 2}) - launch_args = localLauncher.format_launch_args() - assert type(launch_args) == type(list()) - assert launch_args == ["-np", "2"] +def test_format_launcher_args(): + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args={"-np": 2}) + launcher_args = localLauncher.format_launcher_args() + assert launcher_args == ["-np", "2"] @pytest.mark.parametrize( "env_vars", @@ -67,7 +67,7 @@ def test_update_env_null_valued(env_vars): orig_env = {} with pytest.raises(TypeError) as ex: - localLauncher = LaunchSettings(launcher="local", env_vars=orig_env) + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=orig_env) localLauncher.update_env(env_vars) @pytest.mark.parametrize( @@ -82,7 +82,7 @@ def test_update_env_null_valued(env_vars): def test_update_env_initialized(env_vars): """Ensure update of initialized env vars does not overwrite""" orig_env = {"key": "value"} - localLauncher = LaunchSettings(launcher="local", env_vars=orig_env) + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=orig_env) localLauncher.update_env(env_vars) combined_keys = {k for k in env_vars.keys()} @@ -98,8 +98,8 @@ def test_format_env_vars(): "C": "", "D": 12, } - localLauncher = LaunchSettings(launcher="local", env_vars=env_vars) - assert localLauncher.launcher == "local" + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=env_vars) + assert localLauncher.launcher.value == LauncherType.LocalLauncher.value assert isinstance(localLauncher.arg_translator, LocalArgTranslator) assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] @@ -121,8 +121,8 @@ def test_format_env_vars(): pytest.param("set_memory_per_node", (16_000,), id="set_memory_per_node"), pytest.param("set_verbose_launch", (False,), id="set_verbose_launch"), pytest.param("set_quiet_launch", (True,), id="set_quiet_launch"), - pytest.param("set_broadcast", ("/tmp",), id="set_broadcast"), pytest.param("set_walltime", ("00:55:00",), id="set_walltime"), + pytest.param("set_executable_broadcast", ("broad",), id="set_executable_broadcast"), pytest.param("set_binding", ("packed:21",), id="set_binding"), pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), @@ -136,7 +136,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): with caplog.at_level(logging.WARNING): caplog.clear() - localLauncher = LaunchSettings(launcher="local") + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) try: getattr(localLauncher, method)(*params) finally: @@ -145,7 +145,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "local") in rec.msg + and ("method" and "not supported" and "local") in rec.msg ): break else: diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/settings_test/test_mpiLauncher.py index 6e7678f471..4398a64988 100644 --- a/tests/temp_tests/settings_test/test_mpiLauncher.py +++ b/tests/temp_tests/settings_test/test_mpiLauncher.py @@ -3,6 +3,7 @@ import pytest import logging import itertools +from smartsim.settingshold.launchCommand import LauncherType @pytest.mark.parametrize( "l,function,value,result,flag", @@ -11,6 +12,7 @@ *itertools.chain.from_iterable( ( ( + pytest.param(l, "set_walltime", ("100",),"100","timeout",id="set_walltime"), pytest.param(l, "set_task_map", ("taskmap",),"taskmap","map-by",id="set_task_map"), pytest.param(l, "set_cpus_per_task", (2,),2,"cpus-per-proc",id="set_cpus_per_task"), pytest.param(l, "set_cpu_binding_type", ("4",),"4","bind-to",id="set_cpu_binding_type"), @@ -20,31 +22,30 @@ pytest.param(l, "set_hostlist", ("host_A",),"host_A","host",id="set_hostlist_str"), pytest.param(l, "set_hostlist", (["host_A","host_B"],),"host_A,host_B","host",id="set_hostlist_list[str]"), pytest.param(l, "set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","hostfile",id="set_hostlist_from_file"), - pytest.param(l, "set_node_feature", ("P100",),id="set_node_feature"), - pytest.param(l, "set_binding", ("bind",), id="set_binding"), ) - for l in ("mpirun", "orterun", "mpiexec") + for l in ([LauncherType.MpirunLauncher, MpiArgTranslator], [LauncherType.MpiexecLauncher, MpiexecArgTranslator], [LauncherType.OrterunLauncher, OrteArgTranslator]) )) ], ) def test_update_env_initialized(l,function, value, flag, result): - mpiSettings = LaunchSettings(launcher=l) - assert mpiSettings.launcher == l + mpiSettings = LaunchSettings(launcher=l[0]) + assert isinstance(mpiSettings.arg_translator,l[1]) + assert mpiSettings.launcher.value == l[0].value getattr(mpiSettings, function)(*value) assert mpiSettings.launcher_args[flag] == result @pytest.mark.parametrize( "launcher", [ - pytest.param("mpirun", id="format_env"), - pytest.param("orterun", id="format_env"), - pytest.param("mpiexec", id="format_env"), + pytest.param(LauncherType.MpirunLauncher, id="format_env_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="format_env_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="format_env_orterun"), ], ) def test_format_env(launcher): env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} mpiSettings = LaunchSettings(launcher=launcher, env_vars=env_vars) - assert mpiSettings.launcher == launcher + assert mpiSettings.launcher.value == launcher.value formatted = mpiSettings.format_env_vars() result = [ "-x", @@ -57,14 +58,31 @@ def test_format_env(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param("mpirun", id="format_env"), - pytest.param("orterun", id="format_env"), - pytest.param("mpiexec", id="format_env"), + pytest.param(LauncherType.MpirunLauncher, id="format_launcher_args_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="format_launcher_args_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="format_launcher_args_orterun"), + ], +) +def test_format_launcher_args(launcher): + mpiSettings = LaunchSettings(launcher=launcher) + mpiSettings.set_cpus_per_task(1) + mpiSettings.set_tasks(2) + mpiSettings.set_hostlist(["node005", "node006"]) + formatted = mpiSettings.format_launcher_args() + result = ["--cpus-per-proc", "1", "--n", "2", "--host", "node005,node006"] + assert formatted == result + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param(LauncherType.MpirunLauncher, id="set_verbose_launch_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="set_verbose_launch_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="set_verbose_launch_orterun"), ], ) def test_set_verbose_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings.launcher == launcher + assert mpiSettings.launcher.value == launcher.value mpiSettings.set_verbose_launch(True) assert mpiSettings.launcher_args == {'verbose': None} mpiSettings.set_verbose_launch(False) @@ -73,14 +91,14 @@ def test_set_verbose_launch(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param("mpirun", id="format_env"), - pytest.param("orterun", id="format_env"), - pytest.param("mpiexec", id="format_env"), + pytest.param(LauncherType.MpirunLauncher, id="set_quiet_launch_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="set_quiet_launch_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="set_quiet_launch_orterun"), ], ) def test_set_quiet_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings.launcher == launcher + assert mpiSettings.launcher.value == launcher.value mpiSettings.set_quiet_launch(True) assert mpiSettings.launcher_args == {'quiet': None} mpiSettings.set_quiet_launch(False) @@ -89,9 +107,9 @@ def test_set_quiet_launch(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param("mpirun", id="format_env"), - pytest.param("orterun", id="format_env"), - pytest.param("mpiexec", id="format_env"), + pytest.param(LauncherType.MpirunLauncher, id="invalid_hostlist_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="invalid_hostlist_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="invalid_hostlist_orterun"), ], ) def test_invalid_hostlist_format(launcher): @@ -107,19 +125,18 @@ def test_invalid_hostlist_format(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param("mpirun", id="launcher_str_mpirun"), - pytest.param("orterun", id="launcher_str_orterun"), - pytest.param("mpiexec", id="launcher_str_mpiexec"), + pytest.param(LauncherType.MpirunLauncher, id="launcher_str_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="launcher_str_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="launcher_str_orterun"), ], ) def test_launcher_str(launcher): mpiLauncher = LaunchSettings(launcher=launcher) - assert mpiLauncher.launcher_str() == launcher + assert mpiLauncher.launcher_str() == launcher.value @pytest.mark.parametrize( "l,method,params", [ - # Use OpenMPI style settigs for all launchers *itertools.chain.from_iterable( ( ( @@ -128,12 +145,14 @@ def test_launcher_str(launcher): pytest.param(l, "set_cpu_bindings", (1,), id="set_cpu_bindings"), pytest.param(l, "set_memory_per_node", (3000,), id="set_memory_per_node"), pytest.param(l, "set_binding", ("bind",), id="set_binding"), + pytest.param(l, "set_node_feature", ("P100",), id="set_node_feature"), + pytest.param(l, "format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), ) - for l in ("mpirun", "orterun", "mpiexec") + for l in (LauncherType.MpirunLauncher, LauncherType.MpiexecLauncher, LauncherType.OrterunLauncher) )) ], ) -def test_unimplimented_setters_throw_warning(l, caplog, method, params): +def test_unimplimented_methods_throw_warning(l, caplog, method, params): from smartsim.settings.base import logger prev_prop = logger.propagate @@ -150,7 +169,7 @@ def test_unimplimented_setters_throw_warning(l, caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and l) in rec.msg + and (method and "not supported" and l.value) in rec.msg ): break else: From 9c5366413c9a6d646383036ab125cece9042365e Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 16:24:38 -0500 Subject: [PATCH 03/43] lsf and dragon updates --- .../settingshold/translators/launch/dragon.py | 3 +- .../settingshold/translators/launch/lsf.py | 3 +- .../settings_test/test_dragonLauncher.py | 22 ++++++++++---- .../settings_test/test_lsfLauncher.py | 29 +++++++++++++------ 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/smartsim/settingshold/translators/launch/dragon.py b/smartsim/settingshold/translators/launch/dragon.py index 9964e720e7..662d2cfcf2 100644 --- a/smartsim/settingshold/translators/launch/dragon.py +++ b/smartsim/settingshold/translators/launch/dragon.py @@ -1,9 +1,8 @@ from __future__ import annotations -from enum import Enum import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument from ...launchCommand import LauncherType from smartsim.log import get_logger diff --git a/smartsim/settingshold/translators/launch/lsf.py b/smartsim/settingshold/translators/launch/lsf.py index a51bc5f826..0a13f60235 100644 --- a/smartsim/settingshold/translators/launch/lsf.py +++ b/smartsim/settingshold/translators/launch/lsf.py @@ -1,9 +1,8 @@ from __future__ import annotations -from enum import Enum import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from ...launchCommand import LauncherType from smartsim.log import get_logger diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/settings_test/test_dragonLauncher.py index ea9bf9aeec..f8912f834d 100644 --- a/tests/temp_tests/settings_test/test_dragonLauncher.py +++ b/tests/temp_tests/settings_test/test_dragonLauncher.py @@ -1,8 +1,14 @@ from smartsim.settingshold import LaunchSettings from smartsim.settingshold.translators.launch.dragon import DragonArgTranslator +from smartsim.settingshold.launchCommand import LauncherType import pytest import logging - + +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) + assert dragonLauncher.launcher_str() == LauncherType.DragonLauncher.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -11,8 +17,8 @@ ], ) def test_update_env_initialized(function, value, flag, result): - dragonLauncher = LaunchSettings(launcher="dragon") - assert dragonLauncher.launcher == "dragon" + dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) + assert dragonLauncher.launcher.value == LauncherType.DragonLauncher.value assert isinstance(dragonLauncher.arg_translator,DragonArgTranslator) getattr(dragonLauncher, function)(*value) assert dragonLauncher.launcher_args[flag] == result @@ -34,6 +40,12 @@ def test_update_env_initialized(function, value, flag, result): pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), pytest.param("set_walltime", ("10:00:00",),id="set_walltime"), pytest.param("set_node_feature", ("P100",),id="set_node_feature"), + pytest.param("set_memory_per_node", ("1000",),id="set_memory_per_node"), + pytest.param("set_tasks", (2,),id="set_tasks"), + pytest.param("set_binding", ("bind",),id="set_tasks"), + pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + pytest.param("format_launcher_args", (), id="format_launcher_args"), + pytest.param("format_env_vars", (), id="format_env_vars"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): @@ -44,7 +56,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): with caplog.at_level(logging.WARNING): caplog.clear() - dragonLauncher = LaunchSettings(launcher="dragon") + dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) try: getattr(dragonLauncher, method)(*params) finally: @@ -53,7 +65,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "dragon") in rec.msg + and (method and "not supported" and "dragon") in rec.msg ): break else: diff --git a/tests/temp_tests/settings_test/test_lsfLauncher.py b/tests/temp_tests/settings_test/test_lsfLauncher.py index d012ec5dd3..fb8ccb0d6a 100644 --- a/tests/temp_tests/settings_test/test_lsfLauncher.py +++ b/tests/temp_tests/settings_test/test_lsfLauncher.py @@ -1,8 +1,14 @@ from smartsim.settingshold import LaunchSettings from smartsim.settingshold.translators.launch.lsf import JsrunArgTranslator +from smartsim.settingshold.launchCommand import LauncherType import pytest import logging - + +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) + assert lsfLauncher.launcher_str() == LauncherType.LsfLauncher.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -11,16 +17,16 @@ ], ) def test_update_env_initialized(function, value, flag, result): - lsfLauncher = LaunchSettings(launcher="jsrun") - assert lsfLauncher.launcher == "jsrun" + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) + assert lsfLauncher.launcher.value == LauncherType.LsfLauncher.value assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) getattr(lsfLauncher, function)(*value) assert lsfLauncher.launcher_args[flag] == result def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} - lsfLauncher = LaunchSettings(launcher="jsrun", env_vars=env_vars) - assert lsfLauncher.launcher == "jsrun" + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, env_vars=env_vars) + assert lsfLauncher.launcher.value == LauncherType.LsfLauncher.value assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) formatted = lsfLauncher.format_env_vars() assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] @@ -34,8 +40,8 @@ def test_launch_args(): "nrs": 10, "np": 100, } - lsfLauncher = LaunchSettings(launcher="jsrun", launcher_args=launch_args) - assert lsfLauncher.launcher == "jsrun" + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, launcher_args=launch_args) + assert lsfLauncher.launcher.value == "jsrun" formatted = lsfLauncher.format_launcher_args() result = [ "--latency_priority=gpu-gpu", @@ -60,11 +66,16 @@ def test_launch_args(): pytest.param("set_excluded_hosts", (["host_A","host_B"],),id="set_excluded_hosts_list[str]"), pytest.param("set_cpu_bindings", (4,),id="set_cpu_bindings"), pytest.param("set_cpu_bindings", ([4,4],),id="set_cpu_bindings_list[str]"), + pytest.param("set_cpus_per_task", (2,),id="set_cpus_per_task"), pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), pytest.param("set_walltime", ("10:00:00",),id="set_walltime"), pytest.param("set_node_feature", ("P100",),id="set_node_feature"), + pytest.param("set_memory_per_node", ("1000",),id="set_memory_per_node"), + pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_executable_broadcast", ("broad",),id="set_executable_broadcast"), + pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): @@ -75,7 +86,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): with caplog.at_level(logging.WARNING): caplog.clear() - lsfLauncher = LaunchSettings(launcher="jsrun") + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) try: getattr(lsfLauncher, method)(*params) finally: @@ -84,7 +95,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "jsrun") in rec.msg + and (method and "not supported" and "jsrun") in rec.msg ): break else: From 3a9981e1d9b4d70f095fa226976fac33b3f4b125 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 16:40:09 -0500 Subject: [PATCH 04/43] alps updates --- .../settingshold/translators/launch/alps.py | 22 +-- .../settings_test/test_alpsLauncher.py | 123 ++++++++++------ .../settings_test/test_aprunLauncher.py | 138 ------------------ 3 files changed, 92 insertions(+), 191 deletions(-) delete mode 100644 tests/temp_tests/settings_test/test_aprunLauncher.py diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py index 8a451d7804..98e7078cbc 100644 --- a/smartsim/settingshold/translators/launch/alps.py +++ b/smartsim/settingshold/translators/launch/alps.py @@ -20,7 +20,7 @@ def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None :param cpus_per_task: number of cpus to use per task """ - return {"--cpus-per-pe": int(cpus_per_task)} + return {"cpus-per-pe": int(cpus_per_task)} def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: """Set the number of tasks for this job @@ -29,7 +29,7 @@ def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: :param tasks: number of tasks """ - return {"--pes": int(tasks)} + return {"pes": int(tasks)} def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: """Set the number of tasks for this job @@ -38,7 +38,7 @@ def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, No :param tasks_per_node: number of tasks per node """ - return {"--pes-per-node": int(tasks_per_node)} + return {"pes-per-node": int(tasks_per_node)} def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: """Specify the hostlist for this job @@ -54,7 +54,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"--node-list": ",".join(host_list)} + return {"node-list": ",".join(host_list)} def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: """Use the contents of a file to set the node list @@ -63,7 +63,7 @@ def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None :param file_path: Path to the hostlist file """ - return {"--node-list-file": file_path} + return {"node-list-file": file_path} def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: """Specify a list of hosts to exclude for launching this job @@ -79,7 +79,7 @@ def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[St raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"--exclude-node-list": ",".join(host_list)} + return {"exclude-node-list": ",".join(host_list)} def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> t.Union[StringArgument, None]: """Specifies the cores to which MPI processes are bound @@ -90,7 +90,7 @@ def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> t.Union[Strin """ if isinstance(bindings, int): bindings = [bindings] - return {"--cpu-binding": ",".join(str(int(num)) for num in bindings)} + return {"cpu-binding": ",".join(str(int(num)) for num in bindings)} def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument, None]: """Specify the real memory required per node @@ -99,7 +99,7 @@ def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument, N :param memory_per_node: Per PE memory limit in megabytes """ - return {"--memory-per-pe": str(memory_per_node)} + return {"memory-per-pe": str(memory_per_node)} def set_walltime(self, walltime: str) -> t.Union[StringArgument, None]: """Set the walltime of the job @@ -108,7 +108,7 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument, None]: :param walltime: wall time """ - return {"--cpu-time-limit": str(walltime)} + return {"cpu-time-limit": str(walltime)} def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: """Set the job to run in verbose mode @@ -117,7 +117,7 @@ def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict :param verbose: Whether the job should be run verbosely """ - return {"--debug": 7} + return {"debug": 7} def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None],None]: """Set the job to run in quiet mode @@ -126,7 +126,7 @@ def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None],None]: :param quiet: Whether the job should be run quietly """ - return {"--quiet": None} + return {"quiet": None} def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: """Format the environment variables for aprun diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/settings_test/test_alpsLauncher.py index 49a6980c0c..e1b8d6cdd9 100644 --- a/tests/temp_tests/settings_test/test_alpsLauncher.py +++ b/tests/temp_tests/settings_test/test_alpsLauncher.py @@ -1,67 +1,106 @@ +import pytest from smartsim.settingshold import LaunchSettings from smartsim.settingshold.translators.launch.alps import AprunArgTranslator -import pytest - +from smartsim.settingshold.launchCommand import LauncherType +import logging + +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + assert alpsLauncher.launcher_str() == LauncherType.AlpsLauncher.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_cpus_per_task", (4,),4,"--cpus-per-pe",id="set_cpus_per_task"), - pytest.param("set_tasks", (4,),4,"--pes",id="set_tasks"), - pytest.param("set_tasks_per_node", (4,),4,"--pes-per-node",id="set_tasks_per_node"), - pytest.param("set_hostlist", ("host_A",),"host_A","--node-list",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","--node-list",id="set_hostlist_list[str]"), - pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","--node-list-file",id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("host_A",),"host_A","--exclude-node-list",id="set_excluded_hosts_str"), - pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","--exclude-node-list",id="set_excluded_hosts_list[str]"), - pytest.param("set_cpu_bindings", (4,),"4","--cpu-binding",id="set_cpu_bindings"), - pytest.param("set_cpu_bindings", ([4,4],),"4,4","--cpu-binding",id="set_cpu_bindings_list[str]"), - pytest.param("set_memory_per_node", (8000,),8000,"--memory-per-pe",id="set_memory_per_node"), - pytest.param("set_walltime", ("10:00:00",),"10:00:00","--cpu-time-limit",id="set_walltime"), + pytest.param("set_cpus_per_task", (4,),4,"cpus-per-pe",id="set_cpus_per_task"), + pytest.param("set_tasks", (4,),4,"pes",id="set_tasks"), + pytest.param("set_tasks_per_node", (4,),4,"pes-per-node",id="set_tasks_per_node"), + pytest.param("set_hostlist", ("host_A",),"host_A","node-list",id="set_hostlist_str"), + pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","node-list",id="set_hostlist_list[str]"), + pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","node-list-file",id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude-node-list",id="set_excluded_hosts_str"), + pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude-node-list",id="set_excluded_hosts_list[str]"), + pytest.param("set_cpu_bindings", (4,),"4","cpu-binding",id="set_cpu_bindings"), + pytest.param("set_cpu_bindings", ([4,4],),"4,4","cpu-binding",id="set_cpu_bindings_list[str]"), + pytest.param("set_memory_per_node", (8000,),"8000","memory-per-pe",id="set_memory_per_node"), + pytest.param("set_walltime", ("10:00:00",),"10:00:00","cpu-time-limit",id="set_walltime"), + pytest.param("set_verbose_launch", (True,),7,"debug",id="set_verbose_launch"), + pytest.param("set_quiet_launch", (True,),None,"quiet",id="set_quiet_launch"), ], ) def test_update_env_initialized(function, value, flag, result): - alpsLauncher = LaunchSettings(launcher="aprun") - assert alpsLauncher.launcher == "aprun" + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + assert alpsLauncher.launcher.value == LauncherType.AlpsLauncher.value assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) getattr(alpsLauncher, function)(*value) assert alpsLauncher.launcher_args[flag] == result def test_set_verbose_launch(): - slurmLauncher = LaunchSettings(launcher="aprun") - assert slurmLauncher.launcher == "aprun" - assert isinstance(slurmLauncher.arg_translator,AprunArgTranslator) - slurmLauncher.set_verbose_launch(True) - assert slurmLauncher.launcher_args == {'--debug': 7} - slurmLauncher.set_verbose_launch(False) - assert slurmLauncher.launcher_args == {} + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + assert alpsLauncher.launcher.value == LauncherType.AlpsLauncher.value + assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) + alpsLauncher.set_verbose_launch(True) + assert alpsLauncher.launcher_args == {'debug': 7} + alpsLauncher.set_verbose_launch(False) + assert alpsLauncher.launcher_args == {} def test_set_quiet_launch(): - slurmLauncher = LaunchSettings(launcher="aprun") - assert slurmLauncher.launcher == "aprun" - assert isinstance(slurmLauncher.arg_translator,AprunArgTranslator) - slurmLauncher.set_quiet_launch(True) - assert slurmLauncher.launcher_args == {'--quiet': None} - slurmLauncher.set_quiet_launch(False) - assert slurmLauncher.launcher_args == {} + aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + assert aprunLauncher.launcher.value == LauncherType.AlpsLauncher.value + assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) + aprunLauncher.set_quiet_launch(True) + assert aprunLauncher.launcher_args == {'quiet': None} + aprunLauncher.set_quiet_launch(False) + assert aprunLauncher.launcher_args == {} def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} - slurmLauncher = LaunchSettings(launcher="aprun", env_vars=env_vars) - assert slurmLauncher.launcher == "aprun" - formatted = slurmLauncher.format_env_vars() - assert "OMP_NUM_THREADS=20" in formatted - assert "LOGGING=verbose" in formatted - + aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher, env_vars=env_vars) + assert aprunLauncher.launcher.value == LauncherType.AlpsLauncher.value + aprunLauncher.update_env({"OMP_NUM_THREADS": 10}) + formatted = aprunLauncher.format_env_vars() + result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] + assert formatted == result + +def test_aprun_settings(): + aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + aprunLauncher.set_cpus_per_task(2) + aprunLauncher.set_tasks(100) + aprunLauncher.set_tasks_per_node(20) + formatted = aprunLauncher.format_launcher_args() + result = ["--cpus-per-pe=2", "--pes=100", "--pes-per-node=20"] + assert formatted == result + +def test_invalid_hostlist_format(): + """Test invalid hostlist formats""" + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + with pytest.raises(TypeError): + alpsLauncher.set_hostlist(["test",5]) + with pytest.raises(TypeError): + alpsLauncher.set_hostlist([5]) + with pytest.raises(TypeError): + alpsLauncher.set_hostlist(5) + +def test_invalid_exclude_hostlist_format(): + """Test invalid hostlist formats""" + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + with pytest.raises(TypeError): + alpsLauncher.set_excluded_hosts(["test",5]) + with pytest.raises(TypeError): + alpsLauncher.set_excluded_hosts([5]) + with pytest.raises(TypeError): + alpsLauncher.set_excluded_hosts(5) @pytest.mark.parametrize( "method,params", [ - pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), - pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_executable_broadcast"), - pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), + pytest.param("set_nodes", (2,),id="set_nodes"), + pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), + pytest.param("set_node_feature", ("P100",),id="set_node_feature"), pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), pytest.param("set_task_map", ("task:map",), id="set_task_map"), pytest.param("set_binding", ("bind",), id="set_binding"), + pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), ], ) def test_unimplimented_methods_throw_warning(caplog, method, params): @@ -73,16 +112,16 @@ def test_unimplimented_methods_throw_warning(caplog, method, params): with caplog.at_level(logging.WARNING): caplog.clear() - slurmLauncher = LaunchSettings(launcher="slurm") + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) try: - getattr(slurmLauncher, method)(*params) + getattr(alpsLauncher, method)(*params) finally: logger.propagate = prev_prop for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "slurm") in rec.msg + and (method and "not supported" and "aprun") in rec.msg ): break else: diff --git a/tests/temp_tests/settings_test/test_aprunLauncher.py b/tests/temp_tests/settings_test/test_aprunLauncher.py deleted file mode 100644 index c385d09b7d..0000000000 --- a/tests/temp_tests/settings_test/test_aprunLauncher.py +++ /dev/null @@ -1,138 +0,0 @@ -import pytest -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.alps import AprunArgTranslator -import logging - -@pytest.mark.parametrize( - "function,value,result,flag", - [ - pytest.param("set_cpus_per_task", (4,),4,"cpus-per-task",id="set_cpus_per_task"), - pytest.param("set_tasks", (4,),4,"pes",id="set_tasks"), - pytest.param("set_tasks_per_node", (4,),4,"pes-per-node",id="set_tasks_per_node"), - pytest.param("set_hostlist", ("host_A",),"host_A","node-list",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","node-list",id="set_hostlist_list[str]"), - pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","node-list-file",id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude-node-list",id="set_excluded_hosts_str"), - pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude-node-list",id="set_excluded_hosts_list[str]"), - pytest.param("set_cpu_bindings", (4,),"map_cpu:4","cpu-binding",id="set_cpu_bindings"), - pytest.param("set_cpu_bindings", ([4,4],),"map_cpu:4,4","cpu-binding",id="set_cpu_bindings_list[str]"), - pytest.param("set_memory_per_node", (8000,),"8000M","memory-per-pe",id="set_memory_per_node"), - pytest.param("set_walltime", ("10:00:00",),"10:00:00","cpu-time-limit",id="set_walltime"), - pytest.param("set_verbose_launch", (True,),None,"debug",id="set_verbose_launch"), - pytest.param("set_quiet_launch", (True,),None,"quiet",id="set_quiet_launch"), - ], -) -def test_update_env_initialized(function, value, flag, result): - aprunLauncher = LaunchSettings(launcher="aprun") - assert aprunLauncher.launcher == "aprun" - assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) - getattr(aprunLauncher, function)(*value) - assert aprunLauncher.launcher_args[flag] == result - -def test_set_verbose_launch(): - aprunLauncher = LaunchSettings(launcher="aprun") - assert aprunLauncher.launcher == "aprun" - assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) - aprunLauncher.set_verbose_launch(True) - assert aprunLauncher.launcher_args == {'verbose': None} - aprunLauncher.set_verbose_launch(False) - assert aprunLauncher.launcher_args == {} - -def test_set_quiet_launch(): - aprunLauncher = LaunchSettings(launcher="aprun") - assert aprunLauncher.launcher == "aprun" - assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) - aprunLauncher.set_quiet_launch(True) - assert aprunLauncher.launcher_args == {'quiet': None} - aprunLauncher.set_quiet_launch(False) - assert aprunLauncher.launcher_args == {} - -def test_format_env_vars(): - env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} - aprunLauncher = LaunchSettings(launcher="aprun", env_vars=env_vars) - assert aprunLauncher.launcher == "aprun" - aprunLauncher.update_env({"OMP_NUM_THREADS": 10}) - formatted = aprunLauncher.format_env_vars() - result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] - assert formatted == result - -def test_aprun_settings(): - aprunLauncher = LaunchSettings(launcher="aprun") - aprunLauncher.set_cpus_per_task(2) - aprunLauncher.set_tasks(100) - aprunLauncher.set_tasks_per_node(20) - formatted = aprunLauncher.format_run_args() - result = ["--cpus-per-pe=2", "--pes=100", "--pes-per-node=20"] - assert formatted == result - -def test_invalid_hostlist_format(): - """Test invalid hostlist formats""" - slurmLauncher = LaunchSettings(launcher="slurm") - with pytest.raises(TypeError): - slurmLauncher.set_hostlist(["test",5]) - with pytest.raises(TypeError): - slurmLauncher.set_hostlist([5]) - with pytest.raises(TypeError): - slurmLauncher.set_hostlist(5) - -def test_invalid_exclude_hostlist_format(): - """Test invalid hostlist formats""" - slurmLauncher = LaunchSettings(launcher="slurm") - with pytest.raises(TypeError): - slurmLauncher.set_excluded_hosts(["test",5]) - with pytest.raises(TypeError): - slurmLauncher.set_excluded_hosts([5]) - with pytest.raises(TypeError): - slurmLauncher.set_excluded_hosts(5) - -def test_invalid_walltime_format(): - """Test invalid walltime formats""" - slurmLauncher = LaunchSettings(launcher="slurm") - with pytest.raises(ValueError): - slurmLauncher.set_walltime("11:11") - with pytest.raises(ValueError): - slurmLauncher.set_walltime("ss:ss:ss") - with pytest.raises(ValueError): - slurmLauncher.set_walltime("11:ss:ss") - with pytest.raises(ValueError): - slurmLauncher.set_walltime("0s:ss:ss") - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), - pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_broadcast"), - pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("set_task_map", ("task:map",), id="set_task_map"), - pytest.param("set_binding", ("bind",), id="set_binding"), - ], -) -def test_unimplimented_methods_throw_warning(caplog, method, params): - """Test methods not implemented throw warnings""" - from smartsim.settings.base import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - slurmLauncher = LaunchSettings(launcher="slurm") - try: - getattr(slurmLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "slurm") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file From 093722015d19cc7d9bd4c92d4f23d6ee9d152612 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 16:58:52 -0500 Subject: [PATCH 05/43] updates to slurm and pals --- .../settingshold/translators/launch/pals.py | 21 ++++++++++- .../settings_test/test_palsLauncher.py | 30 +++++++++++----- .../settings_test/test_slurmLauncher.py | 35 ++++++++++--------- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py index 1063fdd417..419073e161 100644 --- a/smartsim/settingshold/translators/launch/pals.py +++ b/smartsim/settingshold/translators/launch/pals.py @@ -103,4 +103,23 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> if export_vars: formatted += ["--envlist", ",".join(export_vars)] - return formatted \ No newline at end of file + return formatted + + def format_launcher_args(self, launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + """Return a list of MPI-standard formatted launcher arguments + + :return: list of MPI-standard arguments for these settings + """ + # args launcher uses + args = [] + restricted = ["wdir", "wd"] + + for opt, value in launch_args.items(): + if opt not in restricted: + prefix = "--" + if not value: + args += [prefix + opt] + else: + args += [prefix + opt, str(value)] + + return args \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/settings_test/test_palsLauncher.py index 3a01e0633c..9f1ead80a3 100644 --- a/tests/temp_tests/settings_test/test_palsLauncher.py +++ b/tests/temp_tests/settings_test/test_palsLauncher.py @@ -1,8 +1,14 @@ from smartsim.settingshold import LaunchSettings from smartsim.settingshold.translators.launch.pals import PalsMpiexecArgTranslator +from smartsim.settingshold.launchCommand import LauncherType import pytest import logging - + +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) + assert palsLauncher.launcher_str() == LauncherType.PalsLauncher.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -15,23 +21,23 @@ ], ) def test_update_env_initialized(function, value, flag, result): - palsLauncher = LaunchSettings(launcher="pals") + palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) getattr(palsLauncher, function)(*value) - assert palsLauncher.launcher == "pals" + assert palsLauncher.launcher == LauncherType.PalsLauncher assert isinstance(palsLauncher.arg_translator,PalsMpiexecArgTranslator) assert palsLauncher.launcher_args[flag] == result - assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] + assert palsLauncher.format_launcher_args() == ["--" + flag, str(result)] def test_format_env_vars(): env_vars = {"FOO_VERSION": "3.14", "PATH": None, "LD_LIBRARY_PATH": None} - palsLauncher = LaunchSettings(launcher="pals", env_vars=env_vars) + palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher, env_vars=env_vars) formatted = " ".join(palsLauncher.format_env_vars()) expected = "--env FOO_VERSION=3.14 --envlist PATH,LD_LIBRARY_PATH" assert formatted == expected def test_invalid_hostlist_format(): """Test invalid hostlist formats""" - palsLauncher = LaunchSettings(launcher="pals") + palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) with pytest.raises(TypeError): palsLauncher.set_hostlist(["test",5]) with pytest.raises(TypeError): @@ -42,12 +48,20 @@ def test_invalid_hostlist_format(): @pytest.mark.parametrize( "method,params", [ - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), + pytest.param("set_cpu_bindings", ("bind",), id="set_cpu_bindings"), + pytest.param("set_binding", ("bind",), id="set_binding"), + pytest.param("set_nodes", (2,), id="set_nodes"), pytest.param("set_task_map", ("task:map",), id="set_task_map"), pytest.param("set_cpus_per_task", ("task:map",), id="set_cpus_per_task"), pytest.param("set_quiet_launch", ("task:map",), id="set_quiet_launch"), pytest.param("set_walltime", ("task:map",), id="set_walltime"), pytest.param("set_node_feature", ("P100",),id="set_node_feature"), + pytest.param("set_hostlist_from_file", ("./file",),id="set_hostlist_from_file"), + pytest.param("set_excluded_hosts", ("./file",),id="set_excluded_hosts"), + pytest.param("set_memory_per_node", ("8000",),id="set_memory_per_node"), + pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), + pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), + pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): @@ -58,7 +72,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): with caplog.at_level(logging.WARNING): caplog.clear() - palsLauncher = LaunchSettings(launcher="pals") + palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) try: getattr(palsLauncher, method)(*params) finally: diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py index e72bf38bac..4029aaec62 100644 --- a/tests/temp_tests/settings_test/test_slurmLauncher.py +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -1,5 +1,6 @@ from smartsim.settingshold import LaunchSettings from smartsim.settingshold.translators.launch.slurm import SlurmArgTranslator +from smartsim.settingshold.launchCommand import LauncherType import pytest import logging @@ -25,15 +26,15 @@ ], ) def test_update_env_initialized(function, value, flag, result): - slurmLauncher = LaunchSettings(launcher="slurm") - assert slurmLauncher.launcher == "slurm" + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) getattr(slurmLauncher, function)(*value) assert slurmLauncher.launcher_args[flag] == result def test_set_verbose_launch(): - slurmLauncher = LaunchSettings(launcher="slurm") - assert slurmLauncher.launcher == "slurm" + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) slurmLauncher.set_verbose_launch(True) assert slurmLauncher.launcher_args == {'verbose': None} @@ -41,8 +42,8 @@ def test_set_verbose_launch(): assert slurmLauncher.launcher_args == {} def test_set_quiet_launch(): - slurmLauncher = LaunchSettings(launcher="slurm") - assert slurmLauncher.launcher == "slurm" + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) slurmLauncher.set_quiet_launch(True) assert slurmLauncher.launcher_args == {'quiet': None} @@ -56,8 +57,8 @@ def test_format_env_vars(): "LOGGING": "verbose", "SSKEYIN": "name_0,name_1", } - slurmLauncher = LaunchSettings(launcher="slurm", env_vars=env_vars) - assert slurmLauncher.launcher == "slurm" + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) + assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) formatted = slurmLauncher.format_env_vars() assert "OMP_NUM_THREADS=20" in formatted @@ -67,7 +68,7 @@ def test_format_env_vars(): def test_format_comma_sep_env_vars(): """Test format_comma_sep_env_vars runs correctly""" env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} - slurmLauncher = LaunchSettings(launcher="slurm", env_vars=env_vars) + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) formatted, comma_separated_formatted = slurmLauncher.format_comma_sep_env_vars() assert "OMP_NUM_THREADS" in formatted assert "LOGGING" in formatted @@ -77,7 +78,7 @@ def test_format_comma_sep_env_vars(): def test_srun_settings(): """Test format_launcher_args runs correctly""" - slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) slurmLauncher.set_nodes(5) slurmLauncher.set_cpus_per_task(2) slurmLauncher.set_tasks(100) @@ -95,7 +96,7 @@ def test_srun_launcher_args(): "nodes": 10, "ntasks": 100, } - slurmLauncher = LaunchSettings(launcher="slurm", launcher_args=launcher_args) + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, launcher_args=launcher_args) formatted = slurmLauncher.format_launcher_args() result = [ "--account=A3123", @@ -109,7 +110,7 @@ def test_srun_launcher_args(): def test_invalid_hostlist_format(): """Test invalid hostlist formats""" - slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(TypeError): slurmLauncher.set_hostlist(["test",5]) with pytest.raises(TypeError): @@ -119,7 +120,7 @@ def test_invalid_hostlist_format(): def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" - slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(TypeError): slurmLauncher.set_excluded_hosts(["test",5]) with pytest.raises(TypeError): @@ -129,7 +130,7 @@ def test_invalid_exclude_hostlist_format(): def test_invalid_node_feature_format(): """Test invalid node feature formats""" - slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(TypeError): slurmLauncher.set_node_feature(["test",5]) with pytest.raises(TypeError): @@ -139,7 +140,7 @@ def test_invalid_node_feature_format(): def test_invalid_walltime_format(): """Test invalid walltime formats""" - slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(ValueError): slurmLauncher.set_walltime("11:11") with pytest.raises(ValueError): @@ -166,7 +167,7 @@ def test_unimplimented_methods_throw_warning(caplog, method, params): with caplog.at_level(logging.WARNING): caplog.clear() - slurmLauncher = LaunchSettings(launcher="slurm") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) try: getattr(slurmLauncher, method)(*params) finally: @@ -175,7 +176,7 @@ def test_unimplimented_methods_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "slurm") in rec.msg + and (method and "not supported" and "slurm") in rec.msg ): break else: From 3354e3dd73e2d52fbbfd4dfb7e8ac5fc240ff5cc Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 22:39:11 -0500 Subject: [PATCH 06/43] updates to pbs and slurm --- smartsim/settingshold/batchCommand.py | 6 +- smartsim/settingshold/batchSettings.py | 13 ++- .../translators/batch/__init.__.py | 16 ++-- .../settingshold/translators/batch/slurm.py | 2 +- .../settings_test/test_pbsScheduler.py | 52 ++++++++++- .../settings_test/test_slurmLauncher.py | 7 +- .../settings_test/test_slurmScheduler.py | 92 ++++++++++--------- 7 files changed, 125 insertions(+), 63 deletions(-) diff --git a/smartsim/settingshold/batchCommand.py b/smartsim/settingshold/batchCommand.py index 09c63111d8..4cc04df6bf 100644 --- a/smartsim/settingshold/batchCommand.py +++ b/smartsim/settingshold/batchCommand.py @@ -4,6 +4,6 @@ class SchedulerType(Enum): """ Schedulers that are supported by SmartSim. """ - SlurmLauncher = "sbatch" - PbsLauncher = "qsub" - LsfLauncher = "bsub" \ No newline at end of file + SlurmScheduler = "sbatch" + PbsScheduler = "qsub" + LsfScheduler = "bsub" \ No newline at end of file diff --git a/smartsim/settingshold/batchSettings.py b/smartsim/settingshold/batchSettings.py index b31325e8ec..59776c76dd 100644 --- a/smartsim/settingshold/batchSettings.py +++ b/smartsim/settingshold/batchSettings.py @@ -4,7 +4,7 @@ import copy from smartsim.log import get_logger - +from .batchCommand import SchedulerType from .translators.batch.pbs import QsubBatchArgTranslator from .translators.batch.slurm import SlurmBatchArgTranslator from .translators.batch.lsf import BsubBatchArgTranslator @@ -27,7 +27,7 @@ class SupportedLaunchers(Enum): class BatchSettings(): def __init__( self, - scheduler: str, + scheduler: SchedulerType, scheduler_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, **kwargs: t.Any, @@ -37,7 +37,7 @@ def __init__( 'jsrun' : BsubBatchArgTranslator(), 'qsub' : QsubBatchArgTranslator(), } - if scheduler in scheduler_to_translator: + if scheduler.value in scheduler_to_translator: self.scheduler = scheduler else: raise ValueError(f"'{scheduler}' is not a valid scheduler name.") @@ -47,7 +47,7 @@ def __init__( # TODO check and preporcess launcher_args self.scheduler_args = scheduler_args or {} - self.arg_translator = t.cast(BatchArgTranslator,scheduler_to_translator.get(self.scheduler)) + self.arg_translator = t.cast(BatchArgTranslator,scheduler_to_translator.get(scheduler.value)) @property def scheduler_args(self) -> t.Dict[str, t.Optional[str]]: @@ -65,6 +65,11 @@ def scheduler_args(self, value: t.Dict[str, t.Optional[str]]) -> None: """ self._scheduler_args = copy.deepcopy(value) if value else {} + def scheduler_str(self) -> str: + """ Get the string representation of the scheduler + """ + return self.arg_translator.scheduler_str() + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job diff --git a/smartsim/settingshold/translators/batch/__init.__.py b/smartsim/settingshold/translators/batch/__init.__.py index 6416819f85..c63e374691 100644 --- a/smartsim/settingshold/translators/batch/__init.__.py +++ b/smartsim/settingshold/translators/batch/__init.__.py @@ -1,9 +1,9 @@ -# from .lsf import BsubBatchArgTranslator -# from .pbs import QsubBatchArgTranslator -# from .slurm import SlurmBatchArgTranslator +from .lsf import BsubBatchArgTranslator +from .pbs import QsubBatchArgTranslator +from .slurm import SlurmBatchArgTranslator -# __all__ = [ -# "BsubBatchArgTranslator", -# "QsubBatchArgTranslator", -# "SlurmBatchArgTranslator", -# ] \ No newline at end of file +__all__ = [ + "BsubBatchArgTranslator", + "QsubBatchArgTranslator", + "SlurmBatchArgTranslator", +] \ No newline at end of file diff --git a/smartsim/settingshold/translators/batch/slurm.py b/smartsim/settingshold/translators/batch/slurm.py index d1985c5e4e..24fd4ab43f 100644 --- a/smartsim/settingshold/translators/batch/slurm.py +++ b/smartsim/settingshold/translators/batch/slurm.py @@ -15,7 +15,7 @@ class SlurmBatchArgTranslator(BatchArgTranslator): def scheduler_str(self) -> str: """ Get the string representation of the scheduler """ - return SchedulerType.SlurmLauncher.value + return SchedulerType.SlurmScheduler.value def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: """Set the walltime of the job diff --git a/tests/temp_tests/settings_test/test_pbsScheduler.py b/tests/temp_tests/settings_test/test_pbsScheduler.py index 8075255a09..cbf83b4f90 100644 --- a/tests/temp_tests/settings_test/test_pbsScheduler.py +++ b/tests/temp_tests/settings_test/test_pbsScheduler.py @@ -1,8 +1,14 @@ from smartsim.settingshold import BatchSettings from smartsim.settingshold.translators.batch.pbs import QsubBatchArgTranslator +from smartsim.settingshold.batchCommand import SchedulerType import pytest import logging - + +def test_scheduler_str(): + """Ensure launcher_str returns appropriate value""" + pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) + assert pbsScheduler.scheduler_str() == SchedulerType.PbsScheduler.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -16,12 +22,12 @@ ], ) def test_update_env_initialized(function, value, flag, result): - pbsScheduler = BatchSettings(scheduler="qsub") + pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) getattr(pbsScheduler, function)(*value) assert pbsScheduler.scheduler_args[flag] == result def test_create_pbs_batch(): - pbsScheduler = BatchSettings(scheduler="qsub") + pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) pbsScheduler.set_nodes(1) pbsScheduler.set_walltime("10:00:00") pbsScheduler.set_queue("default") @@ -34,4 +40,42 @@ def test_create_pbs_batch(): "-l walltime=10:00:00", "-q default", "-A myproject", - ] \ No newline at end of file + ] + +# @pytest.mark.parametrize( +# "method,params", +# [ +# pytest.param("set_tasks", (3,), id="set_tasks"), +# pytest.param("set_smts", ("smts",), id="set_smts"), +# pytest.param("set_cpus_per_task", (2,), id="set_cpus_per_task"), +# pytest.param("set_project", ("project",), id="set_project"), +# pytest.param("set_partition", ("project",), id="set_partition"), +# ], +# ) +# def test_unimplimented_setters_throw_warning(caplog, method, params): +# from smartsim.settings.base import logger + +# prev_prop = logger.propagate +# logger.propagate = True + +# with caplog.at_level(logging.WARNING): +# caplog.clear() +# pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) +# try: +# getattr(pbsScheduler, method)(*params) +# finally: +# logger.propagate = prev_prop + +# for rec in caplog.records: +# if ( +# logging.WARNING <= rec.levelno < logging.ERROR +# and (method and "not supported" and "qsub") in rec.msg +# ): +# break +# else: +# pytest.fail( +# ( +# f"No message stating method `{method}` is not " +# "implemented at `warning` level" +# ) +# ) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py index 4029aaec62..654be207a4 100644 --- a/tests/temp_tests/settings_test/test_slurmLauncher.py +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -3,7 +3,12 @@ from smartsim.settingshold.launchCommand import LauncherType import pytest import logging - + +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + assert slurmLauncher.launcher_str() == LauncherType.SlurmLauncher.value + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/settings_test/test_slurmScheduler.py b/tests/temp_tests/settings_test/test_slurmScheduler.py index 021ac6efd0..ddd7005fe2 100644 --- a/tests/temp_tests/settings_test/test_slurmScheduler.py +++ b/tests/temp_tests/settings_test/test_slurmScheduler.py @@ -1,8 +1,14 @@ from smartsim.settingshold import BatchSettings from smartsim.settingshold.translators.batch.slurm import SlurmBatchArgTranslator +from smartsim.settingshold.batchCommand import SchedulerType import pytest import logging - + +def test_scheduler_str(): + """Ensure launcher_str returns appropriate value""" + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + assert slurmScheduler.scheduler_str() == SchedulerType.SlurmScheduler.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -17,13 +23,13 @@ ], ) def test_update_env_initialized(function, value, flag, result): - slurmScheduler = BatchSettings(scheduler="slurm") + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) getattr(slurmScheduler, function)(*value) assert slurmScheduler.scheduler_args[flag] == result def test_create_sbatch(): batch_args = {"exclusive": None, "oversubscribe": None} - slurmScheduler = BatchSettings(scheduler="slurm", scheduler_args=batch_args) + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler, scheduler_args=batch_args) assert isinstance(slurmScheduler.arg_translator, SlurmBatchArgTranslator) #assert slurmScheduler.batch_args["partition"] == "default" args = slurmScheduler.format_batch_args() @@ -42,7 +48,7 @@ def test_launch_args_input_mutation(): key1: val1, key2: val2, } - slurmScheduler = BatchSettings(scheduler="slurm", scheduler_args=default_scheduler_args) + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler, scheduler_args=default_scheduler_args) # Confirm initial values are set assert slurmScheduler.scheduler_args[key0] == val0 @@ -58,52 +64,54 @@ def test_launch_args_input_mutation(): def test_sbatch_settings(): scheduler_args = {"nodes": 1, "time": "10:00:00", "account": "A3123"} - sbatch = BatchSettings(scheduler="slurm",scheduler_args=scheduler_args) - formatted = sbatch.format_batch_args() + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler,scheduler_args=scheduler_args) + formatted = slurmScheduler.format_batch_args() result = ["--nodes=1", "--time=10:00:00", "--account=A3123"] assert formatted == result def test_sbatch_manual(): - sbatch = BatchSettings(scheduler="slurm") - sbatch.set_nodes(5) - sbatch.set_account("A3531") - sbatch.set_walltime("10:00:00") - formatted = sbatch.format_batch_args() + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + slurmScheduler.set_nodes(5) + slurmScheduler.set_account("A3531") + slurmScheduler.set_walltime("10:00:00") + formatted = slurmScheduler.format_batch_args() result = ["--nodes=5", "--account=A3531", "--time=10:00:00"] assert formatted == result -# @pytest.mark.parametrize( -# "method,params", -# [ -# pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), -# pytest.param("set_task_map", ("task:map",), id="set_task_map"), -# ], -# ) -# def test_unimplimented_setters_throw_warning(caplog, method, params): -# from smartsim.settings.base import logger +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_tasks", (3,), id="set_tasks"), + pytest.param("set_smts", ("smts",), id="set_smts"), + pytest.param("set_ncpus", (2,), id="set_ncpus"), + pytest.param("set_project", ("project",), id="set_project"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger -# prev_prop = logger.propagate -# logger.propagate = True + prev_prop = logger.propagate + logger.propagate = True -# with caplog.at_level(logging.WARNING): -# caplog.clear() -# launcher = BatchSettings(scheduler="slurm") -# try: -# getattr(launcher, method)(*params) -# finally: -# logger.propagate = prev_prop + with caplog.at_level(logging.WARNING): + caplog.clear() + slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + try: + getattr(slurmScheduler, method)(*params) + finally: + logger.propagate = prev_prop -# for rec in caplog.records: -# if ( -# logging.WARNING <= rec.levelno < logging.ERROR -# and ("not supported" and "slurm") in rec.msg -# ): -# break -# else: -# pytest.fail( -# ( -# f"No message stating method `{method}` is not " -# "implemented at `warning` level" -# ) -# ) \ No newline at end of file + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and (method and "not supported" and "sbatch") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file From 21455231285669ae1c63875c96dd5a0840aa0e28 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 16 May 2024 22:59:13 -0500 Subject: [PATCH 07/43] delete unused --- smartsim/settingshold/batchSettings.py | 2 +- smartsim/settingshold/launchSettings.py | 5 +- .../settingshold/translators/batch/lsf.py | 4 +- .../settingshold/translators/batch/pbs.py | 5 +- .../settingshold/translators/batch/slurm.py | 3 +- .../translators/batchArgTranslator.py | 5 +- .../settingshold/translators/launch/alps.py | 2 +- .../settingshold/translators/launch/pals.py | 3 +- .../settingshold/translators/launch/slurm.py | 3 +- .../translators/launchArgTranslator.py | 3 +- .../settings_test/test_lsfScheduler.py | 52 +++++++++++++++++-- 11 files changed, 60 insertions(+), 27 deletions(-) diff --git a/smartsim/settingshold/batchSettings.py b/smartsim/settingshold/batchSettings.py index 59776c76dd..8dff43688b 100644 --- a/smartsim/settingshold/batchSettings.py +++ b/smartsim/settingshold/batchSettings.py @@ -34,7 +34,7 @@ def __init__( ) -> None: scheduler_to_translator = { 'sbatch' : SlurmBatchArgTranslator(), - 'jsrun' : BsubBatchArgTranslator(), + 'bsub' : BsubBatchArgTranslator(), 'qsub' : QsubBatchArgTranslator(), } if scheduler.value in scheduler_to_translator: diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index 470468a7e4..a450cba7c6 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -1,5 +1,4 @@ from __future__ import annotations -from enum import Enum import typing as t import copy @@ -14,9 +13,7 @@ from .translators.launch.slurm import SlurmArgTranslator from .translators.launch.dragon import DragonArgTranslator from .translators.launch.local import LocalArgTranslator -from .translators import LaunchArgTranslator - -from .common import process_env_vars, IntegerArgument, StringArgument, FloatArgument +from .translators import LaunchArgTranslator logger = get_logger(__name__) diff --git a/smartsim/settingshold/translators/batch/lsf.py b/smartsim/settingshold/translators/batch/lsf.py index c74cb759f3..84c57e940c 100644 --- a/smartsim/settingshold/translators/batch/lsf.py +++ b/smartsim/settingshold/translators/batch/lsf.py @@ -3,7 +3,7 @@ from enum import Enum import typing as t from ..batchArgTranslator import BatchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from smartsim.log import get_logger from ...batchCommand import SchedulerType @@ -14,7 +14,7 @@ class BsubBatchArgTranslator(BatchArgTranslator): def scheduler_str(self) -> str: """ Get the string representation of the launcher """ - return SchedulerType.LsfLauncher.value + return SchedulerType.LsfScheduler.value def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: """Set the walltime diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py index 5a9935537c..1405ee2668 100644 --- a/smartsim/settingshold/translators/batch/pbs.py +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -1,11 +1,10 @@ from __future__ import annotations -from enum import Enum from copy import deepcopy import typing as t from ..batchArgTranslator import BatchArgTranslator from ....error import SSConfigError -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from smartsim.log import get_logger from ...batchCommand import SchedulerType logger = get_logger(__name__) @@ -15,7 +14,7 @@ class QsubBatchArgTranslator(BatchArgTranslator): def scheduler_str(self) -> str: """ Get the string representation of the launcher """ - return SchedulerType.PbsLauncher.value + return SchedulerType.PbsScheduler.value def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: """Set the number of nodes for this batch job diff --git a/smartsim/settingshold/translators/batch/slurm.py b/smartsim/settingshold/translators/batch/slurm.py index 24fd4ab43f..de265cb0e7 100644 --- a/smartsim/settingshold/translators/batch/slurm.py +++ b/smartsim/settingshold/translators/batch/slurm.py @@ -1,10 +1,9 @@ from __future__ import annotations -from enum import Enum import re import typing as t from ..batchArgTranslator import BatchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from ...batchCommand import SchedulerType from smartsim.log import get_logger diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py index d9945a6c05..442641256f 100644 --- a/smartsim/settingshold/translators/batchArgTranslator.py +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -2,15 +2,12 @@ from abc import ABC, abstractmethod import typing as t -# from launchcommand import LauncherType -from ..common import IntegerArgument, StringArgument, FloatArgument +from ..common import IntegerArgument, StringArgument from smartsim.log import get_logger logger = get_logger(__name__) -from ..common import IntegerArgument, StringArgument, FloatArgument - class BatchArgTranslator(ABC): """Abstract base class that defines all generic scheduler argument methods that are not supported. It is the diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py index 98e7078cbc..bd46d2ea1e 100644 --- a/smartsim/settingshold/translators/launch/alps.py +++ b/smartsim/settingshold/translators/launch/alps.py @@ -1,6 +1,6 @@ from ..launchArgTranslator import LaunchArgTranslator import typing as t -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from ...launchCommand import LauncherType from smartsim.log import get_logger diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py index 419073e161..22a00c6bd0 100644 --- a/smartsim/settingshold/translators/launch/pals.py +++ b/smartsim/settingshold/translators/launch/pals.py @@ -1,9 +1,8 @@ from __future__ import annotations -from enum import Enum import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from ...launchCommand import LauncherType from smartsim.log import get_logger diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py index f1aa21c0fc..bcff5cef22 100644 --- a/smartsim/settingshold/translators/launch/slurm.py +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -1,11 +1,10 @@ from __future__ import annotations -from enum import Enum import typing as t import re import os from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument, FloatArgument +from ...common import IntegerArgument, StringArgument from ...launchCommand import LauncherType from smartsim.log import get_logger diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py index 20f8f27ba4..7854f0bc8d 100644 --- a/smartsim/settingshold/translators/launchArgTranslator.py +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -2,13 +2,12 @@ from abc import ABC, abstractmethod import typing as t -# from launchcommand import LauncherType from smartsim.log import get_logger logger = get_logger(__name__) -from ..common import IntegerArgument, StringArgument, FloatArgument +from ..common import IntegerArgument, StringArgument class LaunchArgTranslator(ABC): diff --git a/tests/temp_tests/settings_test/test_lsfScheduler.py b/tests/temp_tests/settings_test/test_lsfScheduler.py index 26912e22d5..6154bdef4a 100644 --- a/tests/temp_tests/settings_test/test_lsfScheduler.py +++ b/tests/temp_tests/settings_test/test_lsfScheduler.py @@ -1,7 +1,14 @@ from smartsim.settingshold import BatchSettings from smartsim.settingshold.translators.batch.lsf import BsubBatchArgTranslator import pytest - +import logging +from smartsim.settingshold.batchCommand import SchedulerType + +def test_scheduler_str(): + """Ensure launcher_str returns appropriate value""" + lsfScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) + assert lsfScheduler.scheduler_str() == SchedulerType.LsfScheduler.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -17,15 +24,52 @@ ], ) def test_update_env_initialized(function, value, flag, result): - lsfScheduler = BatchSettings(scheduler="jsrun") + lsfScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) getattr(lsfScheduler, function)(*value) assert lsfScheduler.scheduler_args[flag] == result def test_create_bsub(): batch_args = {"core_isolation": None} - lsfScheduler = BatchSettings(scheduler="jsrun", scheduler_args=batch_args) + lsfScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler, scheduler_args=batch_args) lsfScheduler.set_nodes(1) lsfScheduler.set_walltime("10:10:10") lsfScheduler.set_queue("default") args = lsfScheduler.format_batch_args() - assert args == ["-core_isolation", "-nnodes 1", "-W 10:10", "-q default"] \ No newline at end of file + assert args == ["-core_isolation", "-nnodes 1", "-W 10:10", "-q default"] + +# @pytest.mark.parametrize( +# "method,params", +# [ +# pytest.param("set_tasks", (3,), id="set_tasks"), +# pytest.param("set_smts", (10,), id="set_smts"), +# pytest.param("set_ncpus", (2,), id="set_ncpus"), +# pytest.param("set_project", ("project",), id="set_project"), +# ], +# ) +# def test_unimplimented_setters_throw_warning(caplog, method, params): +# from smartsim.settings.base import logger + +# prev_prop = logger.propagate +# logger.propagate = True + +# with caplog.at_level(logging.WARNING): +# caplog.clear() +# slurmScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) +# try: +# getattr(slurmScheduler, method)(*params) +# finally: +# logger.propagate = prev_prop + +# for rec in caplog.records: +# if ( +# logging.WARNING <= rec.levelno < logging.ERROR +# and (method and "not supported" and "bsub") in rec.msg +# ): +# break +# else: +# pytest.fail( +# ( +# f"No message stating method `{method}` is not " +# "implemented at `warning` level" +# ) +# ) \ No newline at end of file From b85d6be26eb1cf92adc53f3aa7b10274de3f207c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Sun, 19 May 2024 22:16:34 -0500 Subject: [PATCH 08/43] run arguments prevented --- smartsim/settingshold/launchSettings.py | 2 +- .../translators/batchArgTranslator.py | 6 ++++ .../settingshold/translators/launch/alps.py | 5 +++- .../settingshold/translators/launch/dragon.py | 3 ++ .../settingshold/translators/launch/local.py | 3 ++ .../settingshold/translators/launch/lsf.py | 3 ++ .../settingshold/translators/launch/mpi.py | 3 ++ .../settingshold/translators/launch/pals.py | 3 ++ .../settingshold/translators/launch/slurm.py | 3 ++ .../translators/launchArgTranslator.py | 7 +++++ .../settings_test/test_alpsLauncher.py | 5 ++++ .../settings_test/test_dragonLauncher.py | 5 ++++ .../settings_test/test_localLauncher.py | 5 ++++ .../settings_test/test_lsfLauncher.py | 5 ++++ .../settings_test/test_mpiLauncher.py | 28 ++++++++++++++++++- .../settings_test/test_palsLauncher.py | 5 ++++ .../settings_test/test_slurmLauncher.py | 5 ++++ 17 files changed, 93 insertions(+), 3 deletions(-) diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index a450cba7c6..7cebd28544 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -46,8 +46,8 @@ def __init__( # TODO check and preporcess launcher_args self.launcher_args = launcher_args or {} - self.arg_translator = t.cast(LaunchArgTranslator,launcher_to_translator.get(launcher.value)) + self._reserved_launch_args = self.arg_translator._set_reserved_launch_args() @property def launcher_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py index 442641256f..4e94f34856 100644 --- a/smartsim/settingshold/translators/batchArgTranslator.py +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -21,6 +21,7 @@ def scheduler_str(self) -> str: """ pass + @abstractmethod def set_account(self, account: str) -> t.Union[StringArgument,None]: """Set the account for this batch job @@ -37,6 +38,7 @@ def set_partition(self, partition: str) -> t.Union[StringArgument,None]: logger.warning(f"set_partition() not supported for {self.scheduler_str()}.") return None + @abstractmethod def set_queue(self, queue: str) -> t.Union[StringArgument,None]: """alias for set_partition @@ -69,6 +71,7 @@ def set_project(self, project: str) -> t.Union[StringArgument,None]: logger.warning(f"set_project() not supported for {self.scheduler_str()}.") return None + @abstractmethod def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: """Set the walltime of the job @@ -77,6 +80,7 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: logger.warning(f"set_walltime() not supported for {self.scheduler_str()}.") return None + @abstractmethod def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: """Set the number of nodes for this batch job @@ -93,6 +97,7 @@ def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None] logger.warning(f"set_cpus_per_task() not supported for {self.scheduler_str()}.") return None + @abstractmethod def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: """Specify the hostlist for this job @@ -102,6 +107,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr logger.warning(f"set_hostlist() not supported for {self.scheduler_str()}.") return None + @abstractmethod def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: """Get the formatted batch arguments for a preview diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py index bd46d2ea1e..90a12cf2c6 100644 --- a/smartsim/settingshold/translators/launch/alps.py +++ b/smartsim/settingshold/translators/launch/alps.py @@ -12,7 +12,10 @@ def launcher_str(self) -> str: """ Get the string representation of the launcher """ return LauncherType.AlpsLauncher.value - + + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {} + def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: """Set the number of cpus to use per task diff --git a/smartsim/settingshold/translators/launch/dragon.py b/smartsim/settingshold/translators/launch/dragon.py index 662d2cfcf2..31f7aa5dfb 100644 --- a/smartsim/settingshold/translators/launch/dragon.py +++ b/smartsim/settingshold/translators/launch/dragon.py @@ -15,6 +15,9 @@ def launcher_str(self) -> str: """ return LauncherType.DragonLauncher.value + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {} + def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: """Set the number of nodes diff --git a/smartsim/settingshold/translators/launch/local.py b/smartsim/settingshold/translators/launch/local.py index 2a24540a07..8a61da81a4 100644 --- a/smartsim/settingshold/translators/launch/local.py +++ b/smartsim/settingshold/translators/launch/local.py @@ -15,6 +15,9 @@ def launcher_str(self) -> str: """ return LauncherType.LocalLauncher.value + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {} + def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: """Build environment variable string diff --git a/smartsim/settingshold/translators/launch/lsf.py b/smartsim/settingshold/translators/launch/lsf.py index 0a13f60235..724b160139 100644 --- a/smartsim/settingshold/translators/launch/lsf.py +++ b/smartsim/settingshold/translators/launch/lsf.py @@ -15,6 +15,9 @@ def launcher_str(self) -> str: """ return LauncherType.LsfLauncher.value + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {"chdir", "h"} + def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: """Set the number of tasks for this job diff --git a/smartsim/settingshold/translators/launch/mpi.py b/smartsim/settingshold/translators/launch/mpi.py index 0bdf8551c8..ea28198e7f 100644 --- a/smartsim/settingshold/translators/launch/mpi.py +++ b/smartsim/settingshold/translators/launch/mpi.py @@ -10,6 +10,9 @@ class _BaseMPIArgTranslator(LaunchArgTranslator): + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {"wd", "wdir"} + def set_task_map(self, task_mapping: str) -> t.Union[StringArgument, None]: """ Set ``mpirun`` task mapping diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py index 22a00c6bd0..612b6a5758 100644 --- a/smartsim/settingshold/translators/launch/pals.py +++ b/smartsim/settingshold/translators/launch/pals.py @@ -15,6 +15,9 @@ def launcher_str(self) -> str: """ return LauncherType.PalsLauncher.value + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {} + def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: """ Specifies the cores to which MPI processes are bound diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py index bcff5cef22..939527e2e2 100644 --- a/smartsim/settingshold/translators/launch/slurm.py +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -17,6 +17,9 @@ def launcher_str(self) -> str: """ Get the string representation of the launcher """ return LauncherType.SlurmLauncher.value + + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + return {"chdir", "D"} def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: """ Set the number of nodes diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py index 7854f0bc8d..e997397b91 100644 --- a/smartsim/settingshold/translators/launchArgTranslator.py +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -23,6 +23,13 @@ def launcher_str(self) -> str: """ pass + def _set_reserved_launch_args(self) -> t.Dict[str,str]: + """ Convert the provide number of nodes into a properly formatted launcher + argument. + """ + logger.warning(f"set_nodes() not supported for {self.launcher_str()}.") + return None + def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: """ Convert the provide number of nodes into a properly formatted launcher argument. diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/settings_test/test_alpsLauncher.py index e1b8d6cdd9..3cf36bf6cc 100644 --- a/tests/temp_tests/settings_test/test_alpsLauncher.py +++ b/tests/temp_tests/settings_test/test_alpsLauncher.py @@ -9,6 +9,11 @@ def test_launcher_str(): alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) assert alpsLauncher.launcher_str() == LauncherType.AlpsLauncher.value +def test_set_reserved_launcher_args(): + """Ensure launcher_str returns appropriate value""" + alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + assert alpsLauncher._reserved_launch_args == {} + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/settings_test/test_dragonLauncher.py index f8912f834d..42f9927c4b 100644 --- a/tests/temp_tests/settings_test/test_dragonLauncher.py +++ b/tests/temp_tests/settings_test/test_dragonLauncher.py @@ -9,6 +9,11 @@ def test_launcher_str(): dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) assert dragonLauncher.launcher_str() == LauncherType.DragonLauncher.value +def test_set_reserved_launcher_args(): + """Ensure launcher_str returns appropriate value""" + dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) + assert dragonLauncher._reserved_launch_args == {} + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/settings_test/test_localLauncher.py b/tests/temp_tests/settings_test/test_localLauncher.py index da9eb37af5..00a838d105 100644 --- a/tests/temp_tests/settings_test/test_localLauncher.py +++ b/tests/temp_tests/settings_test/test_localLauncher.py @@ -9,6 +9,11 @@ def test_launcher_str(): localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) assert localLauncher.launcher_str() == LauncherType.LocalLauncher.value +def test_set_reserved_launcher_args(): + """Ensure launcher_str returns appropriate value""" + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) + assert localLauncher._reserved_launch_args == {} + def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization key0, key1, key2 = "arg0", "arg1", "arg2" diff --git a/tests/temp_tests/settings_test/test_lsfLauncher.py b/tests/temp_tests/settings_test/test_lsfLauncher.py index fb8ccb0d6a..d8da28524a 100644 --- a/tests/temp_tests/settings_test/test_lsfLauncher.py +++ b/tests/temp_tests/settings_test/test_lsfLauncher.py @@ -9,6 +9,11 @@ def test_launcher_str(): lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) assert lsfLauncher.launcher_str() == LauncherType.LsfLauncher.value +def test_set_reserved_launcher_args(): + """Ensure launcher_str returns appropriate value""" + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) + assert lsfLauncher._reserved_launch_args == {"chdir", "h"} + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/settings_test/test_mpiLauncher.py index 4398a64988..9668133172 100644 --- a/tests/temp_tests/settings_test/test_mpiLauncher.py +++ b/tests/temp_tests/settings_test/test_mpiLauncher.py @@ -4,7 +4,33 @@ import logging import itertools from smartsim.settingshold.launchCommand import LauncherType - + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param(LauncherType.MpirunLauncher, id="format_launcher_args_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="format_launcher_args_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="format_launcher_args_orterun"), + ], +) +def test_launcher_str(launcher): + """Ensure launcher_str returns appropriate value""" + mpiSettings = LaunchSettings(launcher=launcher) + assert mpiSettings.launcher_str() == LauncherType.PalsLauncher.value + +@pytest.mark.parametrize( + "launcher", + [ + pytest.param(LauncherType.MpirunLauncher, id="format_launcher_args_mpirun"), + pytest.param(LauncherType.MpiexecLauncher, id="format_launcher_args_mpiexec"), + pytest.param(LauncherType.OrterunLauncher, id="format_launcher_args_orterun"), + ], +) +def test_set_reserved_launcher_args(launcher): + """Ensure launcher_str returns appropriate value""" + mpiSettings = LaunchSettings(launcher=launcher) + assert mpiSettings._reserved_launch_args == {"wd", "wdir"} + @pytest.mark.parametrize( "l,function,value,result,flag", [ diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/settings_test/test_palsLauncher.py index 9f1ead80a3..c914b307ae 100644 --- a/tests/temp_tests/settings_test/test_palsLauncher.py +++ b/tests/temp_tests/settings_test/test_palsLauncher.py @@ -9,6 +9,11 @@ def test_launcher_str(): palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) assert palsLauncher.launcher_str() == LauncherType.PalsLauncher.value +def test_set_reserved_launcher_args(): + """Ensure launcher_str returns appropriate value""" + palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) + assert palsLauncher._reserved_launch_args == {} + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py index 654be207a4..3e6da4ff35 100644 --- a/tests/temp_tests/settings_test/test_slurmLauncher.py +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -9,6 +9,11 @@ def test_launcher_str(): slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) assert slurmLauncher.launcher_str() == LauncherType.SlurmLauncher.value +def test_set_reserved_launcher_args(): + """Ensure launcher_str returns appropriate value""" + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + assert slurmLauncher._reserved_launch_args == {"chdir", "D"} + @pytest.mark.parametrize( "function,value,result,flag", [ From ad26d11f1dbc802bad90963c88606291978fbaa9 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Sun, 19 May 2024 22:24:47 -0500 Subject: [PATCH 09/43] set updated --- smartsim/settingshold/launchSettings.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index 7cebd28544..d90a39cf2f 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -337,12 +337,15 @@ def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: # Store custom arguments in the launcher_args if not isinstance(key, str): raise TypeError("Argument name should be of type str") - # if value is not None and not isinstance(value, str): - # raise TypeError("Argument value should be of type str or None") - #arg = arg.strip().lstrip("-") - # if not condition: - # logger.info(f"Could not set argument '{arg}': condition not met") - # return + key = key.strip().lstrip("-") + if key in self._reserved_launch_args: + logger.warning( + ( + f"Could not set argument '{arg}': " + f"it is a reserved arguement of '{type(self).__name__}'" + ) + ) + return if key in self.launcher_args and arg != self.launcher_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{arg}'") self.launcher_args[key] = arg From 69899de730ffa114e9abd546e84792be5e3d57b6 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Sun, 19 May 2024 22:52:50 -0500 Subject: [PATCH 10/43] set tests added --- smartsim/settingshold/launchSettings.py | 4 +- .../settings_test/test_launchSettings.py | 111 ++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 tests/temp_tests/settings_test/test_launchSettings.py diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index d90a39cf2f..50152b6646 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -341,8 +341,8 @@ def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: if key in self._reserved_launch_args: logger.warning( ( - f"Could not set argument '{arg}': " - f"it is a reserved arguement of '{type(self).__name__}'" + f"Could not set argument '{key}': " + f"it is a reserved argument of '{type(self).__name__}'" ) ) return diff --git a/tests/temp_tests/settings_test/test_launchSettings.py b/tests/temp_tests/settings_test/test_launchSettings.py new file mode 100644 index 0000000000..f7f94127ab --- /dev/null +++ b/tests/temp_tests/settings_test/test_launchSettings.py @@ -0,0 +1,111 @@ +from smartsim.settingshold import LaunchSettings +from smartsim.settingshold.launchCommand import LauncherType +import pytest +import logging + +def test_set_launch_args(): + launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) + launchSettings.set("str", "some-string") + launchSettings.set("nothing", None) + + assert "str" in launchSettings.launcher_args + assert launchSettings.launcher_args["str"] == "some-string" + + assert "nothing" in launchSettings.launcher_args + assert launchSettings.launcher_args["nothing"] is None + + +@pytest.mark.parametrize( + "set_str,val,key", + [ + pytest.param("normal-key", "some-val", "normal-key", id="set string"), + pytest.param("--a-key", "a-value", "a-key", id="strip doulbe dashes"), + pytest.param("-b", "some-str", "b", id="strip single dashes"), + pytest.param(" c ", "some-val", "c", id="strip spaces"), + pytest.param(" --a-mess ", "5", "a-mess", id="strip everything"), + ], +) +def test_set_format_args(set_str, val, key): + launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) + launchSettings.set(set_str, val) + assert launchSettings.launcher_args[key] == val + +def test_set_raises_key_error(): + launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) + with pytest.raises(TypeError): + launchSettings.set(1, "test") + + +@pytest.mark.parametrize( + "launcher,key", + [ + pytest.param(LauncherType.SlurmLauncher, ("chdir",), id="slurm-chdir"), + pytest.param(LauncherType.SlurmLauncher, ("D",), id="slurm-D"), + pytest.param(LauncherType.LsfLauncher, ("chdir",), id="lsf-chdir"), + pytest.param(LauncherType.LsfLauncher, ("h",), id="lsf-h"), + pytest.param(LauncherType.MpiexecLauncher, ("wd",), id="mpiexec-wd"), + pytest.param(LauncherType.OrterunLauncher, ("wd",), id="orte-wd"), + pytest.param(LauncherType.MpirunLauncher, ("wd",), id="mpi-wd"), + pytest.param(LauncherType.MpiexecLauncher, ("wdir",), id="mpiexec-wdir"), + pytest.param(LauncherType.OrterunLauncher, ("wdir",), id="orte-wdir"), + pytest.param(LauncherType.MpirunLauncher, ("wdir",), id="mpi-wdir"), + ], +) +def test_prevent_set_reserved_launch_args(caplog, launcher, key): + """Test methods not implemented throw warnings""" + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + launchSettings = LaunchSettings(launcher=launcher) + try: + getattr(launchSettings, "set")(*key, None) + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and "Could not set argument" in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{key}` is not " + "implemented at `warning` level" + ) + ) + +def test_log_overwrite_set_warning_message(caplog): + """Test methods not implemented throw warnings""" + from smartsim.settings.base import logger + + prev_prop = logger.propagate + logger.propagate = True + + with caplog.at_level(logging.WARNING): + caplog.clear() + launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) + launchSettings.set("test", None) + try: + getattr(launchSettings, "set")("test", "overwritting") + finally: + logger.propagate = prev_prop + + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and "Overwritting argument" in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `test` will be " + "overwritten at `warning` level" + ) + ) \ No newline at end of file From 5f99def319d58e95b5112c52eb39921220dd46f9 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Sun, 19 May 2024 23:29:10 -0500 Subject: [PATCH 11/43] solve some mypy errors --- smartsim/settingshold/common.py | 2 +- .../settingshold/translators/batchArgTranslator.py | 3 +-- smartsim/settingshold/translators/launch/alps.py | 5 +++-- smartsim/settingshold/translators/launch/dragon.py | 4 ++-- smartsim/settingshold/translators/launch/local.py | 4 ++-- smartsim/settingshold/translators/launch/lsf.py | 2 +- smartsim/settingshold/translators/launch/mpi.py | 2 +- smartsim/settingshold/translators/launch/pals.py | 4 ++-- smartsim/settingshold/translators/launch/slurm.py | 2 +- .../settingshold/translators/launchArgTranslator.py | 11 +++++------ 10 files changed, 19 insertions(+), 20 deletions(-) diff --git a/smartsim/settingshold/common.py b/smartsim/settingshold/common.py index 79f1bc40ee..f913d51766 100644 --- a/smartsim/settingshold/common.py +++ b/smartsim/settingshold/common.py @@ -4,7 +4,7 @@ FloatArgument = t.Dict[str, t.Optional[float]] StringArgument = t.Dict[str, t.Optional[str]] -def process_env_vars(env_vars: StringArgument): +def process_env_vars(env_vars: StringArgument) -> None: for key, value in env_vars.items(): if not isinstance(value, str): raise ValueError(f"Value for '{key}' must be a string.") diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py index 4e94f34856..338020b005 100644 --- a/smartsim/settingshold/translators/batchArgTranslator.py +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -113,8 +113,7 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] :return: batch arguments for Sbatch """ - logger.warning(f"format_batch_args() not supported for {self.scheduler_str()}.") - return None + pass def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: """Set the number of tasks for this job diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py index 90a12cf2c6..5548bca8b1 100644 --- a/smartsim/settingshold/translators/launch/alps.py +++ b/smartsim/settingshold/translators/launch/alps.py @@ -1,3 +1,4 @@ +from __future__ import annotations from ..launchArgTranslator import LaunchArgTranslator import typing as t from ...common import IntegerArgument, StringArgument @@ -13,8 +14,8 @@ def launcher_str(self) -> str: """ return LauncherType.AlpsLauncher.value - def _set_reserved_launch_args(self) -> t.Dict[str,str]: - return {} + def _set_reserved_launch_args(self) -> set[str]: + return set() def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: """Set the number of cpus to use per task diff --git a/smartsim/settingshold/translators/launch/dragon.py b/smartsim/settingshold/translators/launch/dragon.py index 31f7aa5dfb..3029a7508a 100644 --- a/smartsim/settingshold/translators/launch/dragon.py +++ b/smartsim/settingshold/translators/launch/dragon.py @@ -15,8 +15,8 @@ def launcher_str(self) -> str: """ return LauncherType.DragonLauncher.value - def _set_reserved_launch_args(self) -> t.Dict[str,str]: - return {} + def _set_reserved_launch_args(self) -> set[str]: + return set() def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: """Set the number of nodes diff --git a/smartsim/settingshold/translators/launch/local.py b/smartsim/settingshold/translators/launch/local.py index 8a61da81a4..d29105f13e 100644 --- a/smartsim/settingshold/translators/launch/local.py +++ b/smartsim/settingshold/translators/launch/local.py @@ -15,8 +15,8 @@ def launcher_str(self) -> str: """ return LauncherType.LocalLauncher.value - def _set_reserved_launch_args(self) -> t.Dict[str,str]: - return {} + def _set_reserved_launch_args(self) -> set[str]: + return set() def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: """Build environment variable string diff --git a/smartsim/settingshold/translators/launch/lsf.py b/smartsim/settingshold/translators/launch/lsf.py index 724b160139..df1487af57 100644 --- a/smartsim/settingshold/translators/launch/lsf.py +++ b/smartsim/settingshold/translators/launch/lsf.py @@ -15,7 +15,7 @@ def launcher_str(self) -> str: """ return LauncherType.LsfLauncher.value - def _set_reserved_launch_args(self) -> t.Dict[str,str]: + def _set_reserved_launch_args(self) -> set[str]: return {"chdir", "h"} def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: diff --git a/smartsim/settingshold/translators/launch/mpi.py b/smartsim/settingshold/translators/launch/mpi.py index ea28198e7f..9fc7078d84 100644 --- a/smartsim/settingshold/translators/launch/mpi.py +++ b/smartsim/settingshold/translators/launch/mpi.py @@ -10,7 +10,7 @@ class _BaseMPIArgTranslator(LaunchArgTranslator): - def _set_reserved_launch_args(self) -> t.Dict[str,str]: + def _set_reserved_launch_args(self) -> set[str]: return {"wd", "wdir"} def set_task_map(self, task_mapping: str) -> t.Union[StringArgument, None]: diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py index 612b6a5758..ad96611ad9 100644 --- a/smartsim/settingshold/translators/launch/pals.py +++ b/smartsim/settingshold/translators/launch/pals.py @@ -15,8 +15,8 @@ def launcher_str(self) -> str: """ return LauncherType.PalsLauncher.value - def _set_reserved_launch_args(self) -> t.Dict[str,str]: - return {} + def _set_reserved_launch_args(self) -> set[str]: + return set() def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: """ Specifies the cores to which MPI processes are bound diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py index 939527e2e2..f17f731bf5 100644 --- a/smartsim/settingshold/translators/launch/slurm.py +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -18,7 +18,7 @@ def launcher_str(self) -> str: """ return LauncherType.SlurmLauncher.value - def _set_reserved_launch_args(self) -> t.Dict[str,str]: + def _set_reserved_launch_args(self) -> set[str]: return {"chdir", "D"} def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py index e997397b91..d6e1fe5ca4 100644 --- a/smartsim/settingshold/translators/launchArgTranslator.py +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -22,13 +22,12 @@ def launcher_str(self) -> str: """ Get the string representation of the launcher """ pass - - def _set_reserved_launch_args(self) -> t.Dict[str,str]: - """ Convert the provide number of nodes into a properly formatted launcher - argument. + + @abstractmethod + def _set_reserved_launch_args(self) -> set[str]: + """ Reserved launch keys per Launcher. """ - logger.warning(f"set_nodes() not supported for {self.launcher_str()}.") - return None + pass def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: """ Convert the provide number of nodes into a properly formatted launcher From 833bcdb70fd0cd5f018527983f34d24d38d10f32 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Sun, 19 May 2024 23:29:21 -0500 Subject: [PATCH 12/43] mypy error --- smartsim/settingshold/translators/batch/pbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py index 1405ee2668..8e9bbf4322 100644 --- a/smartsim/settingshold/translators/batch/pbs.py +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -152,7 +152,7 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No if ncpus := batch_arg_copy.pop("ppn", None): select_command += f":ncpus={ncpus}" if hosts := batch_arg_copy.pop("hostname", None): - hosts_list = ["=".join(hosts)] + hosts_list = ["=".join(str(hosts))] t.cast(str,hosts_list) select_command += f":{'+'.join(hosts_list)}" res += [select_command] From 6edf7b1aa630380d5635671e35aa688bb66ca122 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 20 May 2024 16:19:10 -0500 Subject: [PATCH 13/43] mypy updates --- smartsim/settingshold/batchSettings.py | 46 ++++++++++--------- smartsim/settingshold/common.py | 14 ++++-- smartsim/settingshold/launchSettings.py | 25 ++++++---- .../settingshold/translators/batch/lsf.py | 5 +- .../settingshold/translators/batch/pbs.py | 10 ++-- .../settingshold/translators/batch/slurm.py | 8 ++++ .../translators/batchArgTranslator.py | 15 ++---- .../settingshold/translators/launch/alps.py | 2 + .../settingshold/translators/launch/dragon.py | 2 + .../settingshold/translators/launch/local.py | 2 + .../settingshold/translators/launch/lsf.py | 2 + .../settingshold/translators/launch/mpi.py | 4 +- .../settingshold/translators/launch/pals.py | 2 + .../settingshold/translators/launch/slurm.py | 2 + .../settings_test/test_alpsLauncher.py | 6 +-- .../settings_test/test_batchSettings.py | 17 +++++++ .../settings_test/test_dragonLauncher.py | 2 +- .../settings_test/test_launchSettings.py | 16 ++++++- .../settings_test/test_localLauncher.py | 4 +- .../settings_test/test_mpiLauncher.py | 4 +- .../settings_test/test_palsLauncher.py | 2 +- .../settings_test/test_slurmLauncher.py | 4 +- 22 files changed, 127 insertions(+), 67 deletions(-) create mode 100644 tests/temp_tests/settings_test/test_batchSettings.py diff --git a/smartsim/settingshold/batchSettings.py b/smartsim/settingshold/batchSettings.py index 8dff43688b..c55e5cfb29 100644 --- a/smartsim/settingshold/batchSettings.py +++ b/smartsim/settingshold/batchSettings.py @@ -1,35 +1,25 @@ from __future__ import annotations -from enum import Enum import typing as t import copy -from smartsim.log import get_logger +from smartsim.log import get_logger +from .._core.utils.helpers import fmt_dict +from .common import process_env_vars, process_args, StringArgument from .batchCommand import SchedulerType from .translators.batch.pbs import QsubBatchArgTranslator from .translators.batch.slurm import SlurmBatchArgTranslator from .translators.batch.lsf import BsubBatchArgTranslator -from .translators import BatchArgTranslator - -IntegerArgument_1 = t.Dict[str, t.Optional[int]] -FloatArgument_1 = t.Dict[str, t.Optional[float]] -StringArgument_1 = t.Dict[str, t.Optional[str]] +from .translators import BatchArgTranslator +from .baseSettings import BaseSettings logger = get_logger(__name__) -class SupportedLaunchers(Enum): - """ Launchers that are supported by - SmartSim. - """ - pbs = "qsub" - lsf = "bsub" - slurm = "sbatch" - -class BatchSettings(): +class BatchSettings(BaseSettings): def __init__( self, scheduler: SchedulerType, scheduler_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, - env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, + env_vars: t.Optional[StringArgument] = None, **kwargs: t.Any, ) -> None: scheduler_to_translator = { @@ -42,15 +32,18 @@ def __init__( else: raise ValueError(f"'{scheduler}' is not a valid scheduler name.") - # TODO check and preprocess env_vars + if env_vars: + process_env_vars(env_vars) self.env_vars = env_vars or {} - # TODO check and preporcess launcher_args + if scheduler_args: + process_args(scheduler_args) self.scheduler_args = scheduler_args or {} + self.arg_translator = t.cast(BatchArgTranslator,scheduler_to_translator.get(scheduler.value)) @property - def scheduler_args(self) -> t.Dict[str, t.Optional[str]]: + def scheduler_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: """Retrieve attached batch arguments :returns: attached batch arguments @@ -58,7 +51,7 @@ def scheduler_args(self) -> t.Dict[str, t.Optional[str]]: return self._scheduler_args @scheduler_args.setter - def scheduler_args(self, value: t.Dict[str, t.Optional[str]]) -> None: + def scheduler_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> None: """Attach batch arguments :param value: dictionary of batch arguments @@ -207,4 +200,13 @@ def format_batch_args(self) -> t.List[str]: def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: # Store custom arguments in the launcher_args - self.scheduler_args[key] = arg \ No newline at end of file + self.scheduler_args[key] = arg + + def __str__(self) -> str: # pragma: no-cover + string = "" + string += f"\Scheduler: {self.arg_translator.scheduler_str}" + if self.scheduler_args: + string += f"\Scheduler Arguments:\n{fmt_dict(self.scheduler_args)}" + if self.env_vars: + string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" + return string \ No newline at end of file diff --git a/smartsim/settingshold/common.py b/smartsim/settingshold/common.py index f913d51766..345c57fa5f 100644 --- a/smartsim/settingshold/common.py +++ b/smartsim/settingshold/common.py @@ -1,13 +1,19 @@ import typing as t IntegerArgument = t.Dict[str, t.Optional[int]] -FloatArgument = t.Dict[str, t.Optional[float]] StringArgument = t.Dict[str, t.Optional[str]] def process_env_vars(env_vars: StringArgument) -> None: + """Validate that user passed env vars are of correct type. + """ for key, value in env_vars.items(): - if not isinstance(value, str): - raise ValueError(f"Value for '{key}' must be a string.") + if not isinstance(value, str) and value is not None: + raise TypeError(f"Value for '{key}' must be a string.") -# def process_launcher_args(launcher_args: t.Dict[str, t.Union[str,int,float,None]]): +def process_args(launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> None: + """Validate that user passed launch args and scheduler args are of correct type. + """ + for key, value in launch_args.items(): + if not isinstance(value, (str,int,float)) and value is not None: + raise TypeError(f"Value for '{key}' must be a string.") \ No newline at end of file diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index 50152b6646..40723d819c 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -5,6 +5,7 @@ from smartsim.log import get_logger from .._core.utils.helpers import fmt_dict +from .common import process_env_vars, process_args, StringArgument from .launchCommand import LauncherType from .translators.launch.alps import AprunArgTranslator from .translators.launch.lsf import JsrunArgTranslator @@ -13,16 +14,17 @@ from .translators.launch.slurm import SlurmArgTranslator from .translators.launch.dragon import DragonArgTranslator from .translators.launch.local import LocalArgTranslator -from .translators import LaunchArgTranslator +from .translators import LaunchArgTranslator +from .baseSettings import BaseSettings logger = get_logger(__name__) -class LaunchSettings(): +class LaunchSettings(BaseSettings): def __init__( self, launcher: LauncherType, launcher_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, - env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, + env_vars: t.Optional[StringArgument] = None, **kwargs: t.Any, ) -> None: launcher_to_translator : t.Dict[str,LaunchArgTranslator] = { @@ -40,13 +42,17 @@ def __init__( self.launcher = launcher else: raise ValueError(f"'{launcher}' is not a valid launcher name.") - - #process_env_vars(env_vars) + + if env_vars: + process_env_vars(env_vars) self.env_vars = env_vars or {} - # TODO check and preporcess launcher_args + if launcher_args: + process_args(launcher_args) self.launcher_args = launcher_args or {} + self.arg_translator = t.cast(LaunchArgTranslator,launcher_to_translator.get(launcher.value)) + # Set the reserved launch arguments per LauncherType self._reserved_launch_args = self.arg_translator._set_reserved_launch_args() @property @@ -66,7 +72,7 @@ def launcher_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> No self._launcher_args = copy.deepcopy(value) @property - def env_vars(self) -> t.Dict[str, t.Optional[str]]: + def env_vars(self) -> StringArgument: """Return an immutable list of attached environment variables. :returns: attached environment variables @@ -74,7 +80,7 @@ def env_vars(self) -> t.Dict[str, t.Optional[str]]: return self._env_vars @env_vars.setter - def env_vars(self, value: t.Dict[str, t.Optional[str]]) -> None: + def env_vars(self, value: StringArgument) -> None: """Set the environment variables. :param value: environment variables @@ -352,8 +358,7 @@ def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: def __str__(self) -> str: # pragma: no-cover string = "" - if self.launcher: - string += f"\nLauncher: {self.launcher}" + string += f"\nLauncher: {self.arg_translator.launcher_str}" if self.launcher_args: string += f"\Launch Arguments:\n{fmt_dict(self.launcher_args)}" if self.env_vars: diff --git a/smartsim/settingshold/translators/batch/lsf.py b/smartsim/settingshold/translators/batch/lsf.py index 84c57e940c..5dc18da135 100644 --- a/smartsim/settingshold/translators/batch/lsf.py +++ b/smartsim/settingshold/translators/batch/lsf.py @@ -1,6 +1,5 @@ from __future__ import annotations -from enum import Enum import typing as t from ..batchArgTranslator import BatchArgTranslator from ...common import IntegerArgument, StringArgument @@ -12,7 +11,7 @@ class BsubBatchArgTranslator(BatchArgTranslator): def scheduler_str(self) -> str: - """ Get the string representation of the launcher + """ Get the string representation of the scheduler """ return SchedulerType.LsfScheduler.value @@ -94,6 +93,8 @@ def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: def set_queue(self, queue: str) -> t.Union[StringArgument,None]: """Set the queue for this job + + This sets ``-q`` :param queue: The queue to submit the job on """ diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py index 8e9bbf4322..96954bd1bd 100644 --- a/smartsim/settingshold/translators/batch/pbs.py +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -12,7 +12,7 @@ class QsubBatchArgTranslator(BatchArgTranslator): def scheduler_str(self) -> str: - """ Get the string representation of the launcher + """ Get the string representation of the scheduler """ return SchedulerType.PbsScheduler.value @@ -89,7 +89,7 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] :raises ValueError: if options are supplied without values """ opts, batch_arg_copy = self._create_resource_list(batch_args) - t.cast(t.List,batch_arg_copy) + t.cast(t.List[str],batch_arg_copy) for opt, value in batch_arg_copy.items(): prefix = "-" if not value: @@ -126,16 +126,16 @@ def _sanity_check_resources( for key, value in checked_resources.items(): if not isinstance(key, str): raise TypeError( - f"The type of {key=} is {type(key)}. Only int and str " + f"The type of {key} is {type(key)}. Only int and str " "are allowed." ) if not isinstance(value, (str, int)): raise TypeError( - f"The value associated with {key=} is {type(value)}. Only int " + f"The value associated with {key} is {type(value)}. Only int " "and str are allowed." ) - def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Tuple[t.List[str],t.Dict[str, t.Union[str,int,float,None]]]: self._sanity_check_resources(batch_args) res = [] diff --git a/smartsim/settingshold/translators/batch/slurm.py b/smartsim/settingshold/translators/batch/slurm.py index de265cb0e7..f7cafacd16 100644 --- a/smartsim/settingshold/translators/batch/slurm.py +++ b/smartsim/settingshold/translators/batch/slurm.py @@ -31,6 +31,8 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: """Set the number of nodes for this batch job + + This sets ``--nodes``. :param num_nodes: number of nodes """ @@ -38,6 +40,8 @@ def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: def set_account(self, account: str) -> t.Union[StringArgument,None]: """Set the account for this batch job + + This sets ``--account``. :param account: account id """ @@ -45,6 +49,8 @@ def set_account(self, account: str) -> t.Union[StringArgument,None]: def set_partition(self, partition: str) -> t.Union[StringArgument,None]: """Set the partition for the batch job + + This sets ``--partition``. :param partition: partition name """ @@ -70,6 +76,8 @@ def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None] def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: """Specify the hostlist for this job + + This sets ``--nodelist``. :param host_list: hosts to launch on :raises TypeError: if not str or list of str diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py index 338020b005..f32ed423cc 100644 --- a/smartsim/settingshold/translators/batchArgTranslator.py +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -27,8 +27,7 @@ def set_account(self, account: str) -> t.Union[StringArgument,None]: :param account: account id """ - logger.warning(f"set_account() not supported for {self.scheduler_str()}.") - return None + pass def set_partition(self, partition: str) -> t.Union[StringArgument,None]: """Set the partition for the batch job @@ -46,8 +45,7 @@ def set_queue(self, queue: str) -> t.Union[StringArgument,None]: :param queue: the partition to run the batch job on """ - logger.warning(f"set_queue() not supported for {self.scheduler_str()}.") - return None + pass def set_smts(self, smts: int) -> t.Union[IntegerArgument,None]: """Set SMTs @@ -77,8 +75,7 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: :param walltime: wall time """ - logger.warning(f"set_walltime() not supported for {self.scheduler_str()}.") - return None + pass @abstractmethod def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: @@ -86,8 +83,7 @@ def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: :param num_nodes: number of nodes """ - logger.warning(f"set_nodes() not supported for {self.scheduler_str()}.") - return None + pass def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: """Set the number of cpus to use per task @@ -104,8 +100,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr :param host_list: hosts to launch on :raises TypeError: if not str or list of str """ - logger.warning(f"set_hostlist() not supported for {self.scheduler_str()}.") - return None + pass @abstractmethod def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py index 5548bca8b1..803aacb897 100644 --- a/smartsim/settingshold/translators/launch/alps.py +++ b/smartsim/settingshold/translators/launch/alps.py @@ -15,6 +15,8 @@ def launcher_str(self) -> str: return LauncherType.AlpsLauncher.value def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return set() def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: diff --git a/smartsim/settingshold/translators/launch/dragon.py b/smartsim/settingshold/translators/launch/dragon.py index 3029a7508a..fdf027f1a3 100644 --- a/smartsim/settingshold/translators/launch/dragon.py +++ b/smartsim/settingshold/translators/launch/dragon.py @@ -16,6 +16,8 @@ def launcher_str(self) -> str: return LauncherType.DragonLauncher.value def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return set() def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: diff --git a/smartsim/settingshold/translators/launch/local.py b/smartsim/settingshold/translators/launch/local.py index d29105f13e..c1124ceb2b 100644 --- a/smartsim/settingshold/translators/launch/local.py +++ b/smartsim/settingshold/translators/launch/local.py @@ -16,6 +16,8 @@ def launcher_str(self) -> str: return LauncherType.LocalLauncher.value def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return set() def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: diff --git a/smartsim/settingshold/translators/launch/lsf.py b/smartsim/settingshold/translators/launch/lsf.py index df1487af57..7e7469c5d9 100644 --- a/smartsim/settingshold/translators/launch/lsf.py +++ b/smartsim/settingshold/translators/launch/lsf.py @@ -16,6 +16,8 @@ def launcher_str(self) -> str: return LauncherType.LsfLauncher.value def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return {"chdir", "h"} def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: diff --git a/smartsim/settingshold/translators/launch/mpi.py b/smartsim/settingshold/translators/launch/mpi.py index 9fc7078d84..1097d758ce 100644 --- a/smartsim/settingshold/translators/launch/mpi.py +++ b/smartsim/settingshold/translators/launch/mpi.py @@ -11,6 +11,8 @@ class _BaseMPIArgTranslator(LaunchArgTranslator): def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return {"wd", "wdir"} def set_task_map(self, task_mapping: str) -> t.Union[StringArgument, None]: @@ -111,7 +113,7 @@ def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict """ return {"verbose": None} - def set_walltime(self, walltime: str) -> None: + def set_walltime(self, walltime: str) -> t.Union[StringArgument, None]: """Set the maximum number of seconds that a job will run This sets ``--timeout`` diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py index ad96611ad9..ca5afb0550 100644 --- a/smartsim/settingshold/translators/launch/pals.py +++ b/smartsim/settingshold/translators/launch/pals.py @@ -16,6 +16,8 @@ def launcher_str(self) -> str: return LauncherType.PalsLauncher.value def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return set() def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py index f17f731bf5..6985843d3e 100644 --- a/smartsim/settingshold/translators/launch/slurm.py +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -19,6 +19,8 @@ def launcher_str(self) -> str: return LauncherType.SlurmLauncher.value def _set_reserved_launch_args(self) -> set[str]: + """ Return reserved launch arguments. + """ return {"chdir", "D"} def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/settings_test/test_alpsLauncher.py index 3cf36bf6cc..71a2a9370e 100644 --- a/tests/temp_tests/settings_test/test_alpsLauncher.py +++ b/tests/temp_tests/settings_test/test_alpsLauncher.py @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert alpsLauncher._reserved_launch_args == {} + assert alpsLauncher._reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", @@ -59,10 +59,10 @@ def test_set_quiet_launch(): assert aprunLauncher.launcher_args == {} def test_format_env_vars(): - env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} + env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher, env_vars=env_vars) assert aprunLauncher.launcher.value == LauncherType.AlpsLauncher.value - aprunLauncher.update_env({"OMP_NUM_THREADS": 10}) + aprunLauncher.update_env({"OMP_NUM_THREADS": "10"}) formatted = aprunLauncher.format_env_vars() result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] assert formatted == result diff --git a/tests/temp_tests/settings_test/test_batchSettings.py b/tests/temp_tests/settings_test/test_batchSettings.py new file mode 100644 index 0000000000..0c146f0482 --- /dev/null +++ b/tests/temp_tests/settings_test/test_batchSettings.py @@ -0,0 +1,17 @@ +from smartsim.settingshold import BatchSettings +from smartsim.settingshold.batchCommand import SchedulerType +import pytest + +def test_incorrect_env_var_type(): + with pytest.raises(TypeError): + _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, env_vars={"str": 2}) + with pytest.raises(TypeError): + _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, env_vars={"str": 2.0}) + with pytest.raises(TypeError): + _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, env_vars={"str": "str", "str": 2.0}) + +def test_incorrect_scheduler_arg_type(): + with pytest.raises(TypeError): + _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, launcher_args={"str": [1,2]}) + with pytest.raises(TypeError): + _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, launcher_args={"str": SchedulerType.SlurmScheduler}) \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/settings_test/test_dragonLauncher.py index 42f9927c4b..430410222c 100644 --- a/tests/temp_tests/settings_test/test_dragonLauncher.py +++ b/tests/temp_tests/settings_test/test_dragonLauncher.py @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert dragonLauncher._reserved_launch_args == {} + assert dragonLauncher._reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", diff --git a/tests/temp_tests/settings_test/test_launchSettings.py b/tests/temp_tests/settings_test/test_launchSettings.py index f7f94127ab..2491b539ec 100644 --- a/tests/temp_tests/settings_test/test_launchSettings.py +++ b/tests/temp_tests/settings_test/test_launchSettings.py @@ -14,7 +14,6 @@ def test_set_launch_args(): assert "nothing" in launchSettings.launcher_args assert launchSettings.launcher_args["nothing"] is None - @pytest.mark.parametrize( "set_str,val,key", [ @@ -34,7 +33,20 @@ def test_set_raises_key_error(): launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) with pytest.raises(TypeError): launchSettings.set(1, "test") - + +def test_incorrect_env_var_type(): + with pytest.raises(TypeError): + _ = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars={"str": 2}) + with pytest.raises(TypeError): + _ = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars={"str": 2.0}) + with pytest.raises(TypeError): + _ = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars={"str": "str", "str": 2.0}) + +def test_incorrect_launch_arg_type(): + with pytest.raises(TypeError): + _ = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args={"str": [1,2]}) + with pytest.raises(TypeError): + _ = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args={"str": LauncherType.LocalLauncher}) @pytest.mark.parametrize( "launcher,key", diff --git a/tests/temp_tests/settings_test/test_localLauncher.py b/tests/temp_tests/settings_test/test_localLauncher.py index 00a838d105..1bf157b56a 100644 --- a/tests/temp_tests/settings_test/test_localLauncher.py +++ b/tests/temp_tests/settings_test/test_localLauncher.py @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) - assert localLauncher._reserved_launch_args == {} + assert localLauncher._reserved_launch_args == set() def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization @@ -101,7 +101,7 @@ def test_format_env_vars(): "A": "a", "B": None, "C": "", - "D": 12, + "D": "12", } localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=env_vars) assert localLauncher.launcher.value == LauncherType.LocalLauncher.value diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/settings_test/test_mpiLauncher.py index 9668133172..e6e71bce7a 100644 --- a/tests/temp_tests/settings_test/test_mpiLauncher.py +++ b/tests/temp_tests/settings_test/test_mpiLauncher.py @@ -68,8 +68,8 @@ def test_update_env_initialized(l,function, value, flag, result): pytest.param(LauncherType.OrterunLauncher, id="format_env_orterun"), ], ) -def test_format_env(launcher): - env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose"} +def test_format_env_vars(launcher): + env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} mpiSettings = LaunchSettings(launcher=launcher, env_vars=env_vars) assert mpiSettings.launcher.value == launcher.value formatted = mpiSettings.format_env_vars() diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/settings_test/test_palsLauncher.py index c914b307ae..6f648cf3e3 100644 --- a/tests/temp_tests/settings_test/test_palsLauncher.py +++ b/tests/temp_tests/settings_test/test_palsLauncher.py @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) - assert palsLauncher._reserved_launch_args == {} + assert palsLauncher._reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py index 3e6da4ff35..a4aada88f3 100644 --- a/tests/temp_tests/settings_test/test_slurmLauncher.py +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -63,7 +63,7 @@ def test_set_quiet_launch(): def test_format_env_vars(): """Test format_env_vars runs correctly""" env_vars={ - "OMP_NUM_THREADS": 20, + "OMP_NUM_THREADS": "20", "LOGGING": "verbose", "SSKEYIN": "name_0,name_1", } @@ -77,7 +77,7 @@ def test_format_env_vars(): def test_format_comma_sep_env_vars(): """Test format_comma_sep_env_vars runs correctly""" - env_vars = {"OMP_NUM_THREADS": 20, "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} + env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) formatted, comma_separated_formatted = slurmLauncher.format_comma_sep_env_vars() assert "OMP_NUM_THREADS" in formatted From 7e967ccac1a950432b22d8f945d4d99afca2b76a Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 20 May 2024 16:43:46 -0500 Subject: [PATCH 14/43] het group method added --- smartsim/settingshold/launchSettings.py | 12 +++++++++++ .../settingshold/translators/launch/slurm.py | 20 +++++++++++++++++++ .../translators/launchArgTranslator.py | 10 ++++++++++ .../settings_test/test_alpsLauncher.py | 1 + .../settings_test/test_dragonLauncher.py | 1 + .../settings_test/test_localLauncher.py | 1 + .../settings_test/test_lsfLauncher.py | 1 + .../settings_test/test_mpiLauncher.py | 1 + .../settings_test/test_palsLauncher.py | 1 + .../settings_test/test_slurmLauncher.py | 13 +++++++++++- 10 files changed, 60 insertions(+), 1 deletion(-) diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index 40723d819c..1347c5e2da 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -261,6 +261,18 @@ def set_task_map(self, task_mapping: str) -> None: for key, value in args.items(): self.set(key, value) + def set_het_group(self, het_group: t.Iterable[int]) -> None: + """Set the heterogeneous group for this job + + this sets `--het-group` + + :param het_group: list of heterogeneous groups + """ + args = self.arg_translator.set_het_group(het_group) + if args: + for key, value in args.items(): + self.set(key, value) + def set_verbose_launch(self, verbose: bool) -> None: """Set the job to run in verbose mode diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py index 6985843d3e..685b8a6631 100644 --- a/smartsim/settingshold/translators/launch/slurm.py +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -155,6 +155,26 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: else: raise ValueError("Invalid walltime format. Please use 'HH:MM:SS' format.") + def set_het_group(self, het_group: t.Iterable[int]) -> t.Union[StringArgument,None]: + """Set the heterogeneous group for this job + + this sets `--het-group` + + :param het_group: list of heterogeneous groups + """ + het_size_env = os.getenv("SLURM_HET_SIZE") + if het_size_env is None: + msg = "Requested to set het group, but the allocation is not a het job" + raise ValueError(msg) + het_size = int(het_size_env) + if any(group >= het_size for group in het_group): + msg = ( + f"Het group {max(het_group)} requested, " + f"but max het group in allocation is {het_size-1}" + ) + raise ValueError(msg) + return {"het-group": ",".join(str(group) for group in het_group)} + def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: """ Set the job to run in verbose mode diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py index d6e1fe5ca4..b02a43a58b 100644 --- a/smartsim/settingshold/translators/launchArgTranslator.py +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -136,6 +136,16 @@ def set_task_map(self, task_mapping: str) -> t.Union[StringArgument,None]: logger.warning(f"set_task_map() not supported for {self.launcher_str()}.") return None + def set_het_group(self, het_group: t.Iterable[int]) -> t.Union[StringArgument, None]: + """Set the heterogeneous group for this job + + this sets `--het-group` + + :param het_group: list of heterogeneous groups + """ + logger.warning(f"set_het_group() not supported for {self.launcher_str()}.") + return None + def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str, None], None]: """Set the job to run in quiet mode diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/settings_test/test_alpsLauncher.py index 71a2a9370e..cf821ce571 100644 --- a/tests/temp_tests/settings_test/test_alpsLauncher.py +++ b/tests/temp_tests/settings_test/test_alpsLauncher.py @@ -106,6 +106,7 @@ def test_invalid_exclude_hostlist_format(): pytest.param("set_task_map", ("task:map",), id="set_task_map"), pytest.param("set_binding", ("bind",), id="set_binding"), pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), ], ) def test_unimplimented_methods_throw_warning(caplog, method, params): diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/settings_test/test_dragonLauncher.py index 430410222c..e4506bbc64 100644 --- a/tests/temp_tests/settings_test/test_dragonLauncher.py +++ b/tests/temp_tests/settings_test/test_dragonLauncher.py @@ -51,6 +51,7 @@ def test_update_env_initialized(function, value, flag, result): pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), pytest.param("format_launcher_args", (), id="format_launcher_args"), pytest.param("format_env_vars", (), id="format_env_vars"), + pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): diff --git a/tests/temp_tests/settings_test/test_localLauncher.py b/tests/temp_tests/settings_test/test_localLauncher.py index 1bf157b56a..09365cfd79 100644 --- a/tests/temp_tests/settings_test/test_localLauncher.py +++ b/tests/temp_tests/settings_test/test_localLauncher.py @@ -131,6 +131,7 @@ def test_format_env_vars(): pytest.param("set_binding", ("packed:21",), id="set_binding"), pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): diff --git a/tests/temp_tests/settings_test/test_lsfLauncher.py b/tests/temp_tests/settings_test/test_lsfLauncher.py index d8da28524a..46fbb0cb41 100644 --- a/tests/temp_tests/settings_test/test_lsfLauncher.py +++ b/tests/temp_tests/settings_test/test_lsfLauncher.py @@ -81,6 +81,7 @@ def test_launch_args(): pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), pytest.param("set_executable_broadcast", ("broad",),id="set_executable_broadcast"), pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/settings_test/test_mpiLauncher.py index e6e71bce7a..e8cdddad91 100644 --- a/tests/temp_tests/settings_test/test_mpiLauncher.py +++ b/tests/temp_tests/settings_test/test_mpiLauncher.py @@ -173,6 +173,7 @@ def test_launcher_str(launcher): pytest.param(l, "set_binding", ("bind",), id="set_binding"), pytest.param(l, "set_node_feature", ("P100",), id="set_node_feature"), pytest.param(l, "format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + pytest.param(l, "set_het_group", ([1,2,3,4],), id="set_het_group"), ) for l in (LauncherType.MpirunLauncher, LauncherType.MpiexecLauncher, LauncherType.OrterunLauncher) )) diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/settings_test/test_palsLauncher.py index 6f648cf3e3..13ad3ff979 100644 --- a/tests/temp_tests/settings_test/test_palsLauncher.py +++ b/tests/temp_tests/settings_test/test_palsLauncher.py @@ -67,6 +67,7 @@ def test_invalid_hostlist_format(): pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), + pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py index a4aada88f3..1e09c23889 100644 --- a/tests/temp_tests/settings_test/test_slurmLauncher.py +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -195,4 +195,15 @@ def test_unimplimented_methods_throw_warning(caplog, method, params): f"No message stating method `{method}` is not " "implemented at `warning` level" ) - ) \ No newline at end of file + ) + +def test_set_het_groups(monkeypatch): + """Test ability to set one or more het groups to run setting""" + monkeypatch.setenv("SLURM_HET_SIZE", "4") + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher.set_het_group([1]) + assert slurmLauncher.launcher_args["het-group"] == "1" + slurmLauncher.set_het_group([3, 2]) + assert slurmLauncher.launcher_args["het-group"] == "3,2" + with pytest.raises(ValueError): + slurmLauncher.set_het_group([4]) \ No newline at end of file From 5d18460d8cb81c96a09df71ebe624bf1db46cfbc Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 20 May 2024 17:15:41 -0500 Subject: [PATCH 15/43] resources --- .../settingshold/translators/batch/pbs.py | 5 +- .../settings_test/test_pbsScheduler.py | 68 +++++++++---------- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py index 96954bd1bd..7e012b0af7 100644 --- a/smartsim/settingshold/translators/batch/pbs.py +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -105,7 +105,6 @@ def _sanity_check_resources( Note: For PBS Pro, nodes is equivalent to 'select' and 'place' so they are not quite synonyms. Here we assume that """ - # Note: isinstance check here to avoid collision with default checked_resources = batch_args has_select = checked_resources.get("select", None) @@ -139,7 +138,7 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No self._sanity_check_resources(batch_args) res = [] - batch_arg_copy = deepcopy(batch_args) + batch_arg_copy = batch_args # Construct the basic select/nodes statement if select := batch_arg_copy.pop("select", None): select_command = f"-l select={select}" @@ -159,7 +158,7 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No if walltime := batch_arg_copy.pop("walltime", None): res += [f"-l walltime={walltime}"] - # All other "standard" resource specs + # # All other "standard" resource specs # for resource, value in batch_arg_copy.items(): # res += [f"-l {resource}={value}"] diff --git a/tests/temp_tests/settings_test/test_pbsScheduler.py b/tests/temp_tests/settings_test/test_pbsScheduler.py index cbf83b4f90..614ec181ce 100644 --- a/tests/temp_tests/settings_test/test_pbsScheduler.py +++ b/tests/temp_tests/settings_test/test_pbsScheduler.py @@ -42,40 +42,40 @@ def test_create_pbs_batch(): "-A myproject", ] -# @pytest.mark.parametrize( -# "method,params", -# [ -# pytest.param("set_tasks", (3,), id="set_tasks"), -# pytest.param("set_smts", ("smts",), id="set_smts"), -# pytest.param("set_cpus_per_task", (2,), id="set_cpus_per_task"), -# pytest.param("set_project", ("project",), id="set_project"), -# pytest.param("set_partition", ("project",), id="set_partition"), -# ], -# ) -# def test_unimplimented_setters_throw_warning(caplog, method, params): -# from smartsim.settings.base import logger +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_tasks", (3,), id="set_tasks"), + pytest.param("set_smts", ("smts",), id="set_smts"), + pytest.param("set_cpus_per_task", (2,), id="set_cpus_per_task"), + pytest.param("set_project", ("project",), id="set_project"), + pytest.param("set_partition", ("project",), id="set_partition"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger -# prev_prop = logger.propagate -# logger.propagate = True + prev_prop = logger.propagate + logger.propagate = True -# with caplog.at_level(logging.WARNING): -# caplog.clear() -# pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) -# try: -# getattr(pbsScheduler, method)(*params) -# finally: -# logger.propagate = prev_prop + with caplog.at_level(logging.WARNING): + caplog.clear() + pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) + try: + getattr(pbsScheduler, method)(*params) + finally: + logger.propagate = prev_prop -# for rec in caplog.records: -# if ( -# logging.WARNING <= rec.levelno < logging.ERROR -# and (method and "not supported" and "qsub") in rec.msg -# ): -# break -# else: -# pytest.fail( -# ( -# f"No message stating method `{method}` is not " -# "implemented at `warning` level" -# ) -# ) \ No newline at end of file + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and (method and "not supported" and "qsub") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file From 554dc581f8f334516313759eb2e1aba6186f6f6b Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 20 May 2024 17:54:08 -0500 Subject: [PATCH 16/43] fixing up tests --- .../settingshold/translators/batch/pbs.py | 2 +- .../settings_test/test_alpsLauncher.py | 2 +- .../settings_test/test_dragonLauncher.py | 2 +- .../settings_test/test_lsfLauncher.py | 2 +- .../settings_test/test_mpiLauncher.py | 2 +- .../settings_test/test_palsLauncher.py | 2 +- .../settings_test/test_slurmLauncher.py | 36 +++++++++++++++++-- 7 files changed, 39 insertions(+), 9 deletions(-) diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py index 7e012b0af7..716f05caff 100644 --- a/smartsim/settingshold/translators/batch/pbs.py +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -138,7 +138,7 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No self._sanity_check_resources(batch_args) res = [] - batch_arg_copy = batch_args + batch_arg_copy = deepcopy(batch_args) # Construct the basic select/nodes statement if select := batch_arg_copy.pop("select", None): select_command = f"-l select={select}" diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/settings_test/test_alpsLauncher.py index cf821ce571..ff1213499d 100644 --- a/tests/temp_tests/settings_test/test_alpsLauncher.py +++ b/tests/temp_tests/settings_test/test_alpsLauncher.py @@ -33,7 +33,7 @@ def test_set_reserved_launcher_args(): pytest.param("set_quiet_launch", (True,),None,"quiet",id="set_quiet_launch"), ], ) -def test_update_env_initialized(function, value, flag, result): +def test_alps_class_methods(function, value, flag, result): alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) assert alpsLauncher.launcher.value == LauncherType.AlpsLauncher.value assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/settings_test/test_dragonLauncher.py index e4506bbc64..9f582d00e6 100644 --- a/tests/temp_tests/settings_test/test_dragonLauncher.py +++ b/tests/temp_tests/settings_test/test_dragonLauncher.py @@ -21,7 +21,7 @@ def test_set_reserved_launcher_args(): pytest.param("set_tasks_per_node", (2,),2,"tasks-per-node",id="set_tasks_per_node"), ], ) -def test_update_env_initialized(function, value, flag, result): +def test_dragon_class_methods(function, value, flag, result): dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) assert dragonLauncher.launcher.value == LauncherType.DragonLauncher.value assert isinstance(dragonLauncher.arg_translator,DragonArgTranslator) diff --git a/tests/temp_tests/settings_test/test_lsfLauncher.py b/tests/temp_tests/settings_test/test_lsfLauncher.py index 46fbb0cb41..9b90649166 100644 --- a/tests/temp_tests/settings_test/test_lsfLauncher.py +++ b/tests/temp_tests/settings_test/test_lsfLauncher.py @@ -21,7 +21,7 @@ def test_set_reserved_launcher_args(): pytest.param("set_binding", ("packed:21",),"packed:21","bind",id="set_binding"), ], ) -def test_update_env_initialized(function, value, flag, result): +def test_lsf_class_methods(function, value, flag, result): lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) assert lsfLauncher.launcher.value == LauncherType.LsfLauncher.value assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/settings_test/test_mpiLauncher.py index e8cdddad91..7c1615706b 100644 --- a/tests/temp_tests/settings_test/test_mpiLauncher.py +++ b/tests/temp_tests/settings_test/test_mpiLauncher.py @@ -53,7 +53,7 @@ def test_set_reserved_launcher_args(launcher): )) ], ) -def test_update_env_initialized(l,function, value, flag, result): +def test_mpi_class_methods(l,function, value, flag, result): mpiSettings = LaunchSettings(launcher=l[0]) assert isinstance(mpiSettings.arg_translator,l[1]) assert mpiSettings.launcher.value == l[0].value diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/settings_test/test_palsLauncher.py index 13ad3ff979..904c3c54fb 100644 --- a/tests/temp_tests/settings_test/test_palsLauncher.py +++ b/tests/temp_tests/settings_test/test_palsLauncher.py @@ -25,7 +25,7 @@ def test_set_reserved_launcher_args(): pytest.param("set_executable_broadcast", ("broadcast",),"broadcast","transfer",id="set_executable_broadcast"), ], ) -def test_update_env_initialized(function, value, flag, result): +def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) getattr(palsLauncher, function)(*value) assert palsLauncher.launcher == LauncherType.PalsLauncher diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/settings_test/test_slurmLauncher.py index 1e09c23889..50c726bc37 100644 --- a/tests/temp_tests/settings_test/test_slurmLauncher.py +++ b/tests/temp_tests/settings_test/test_slurmLauncher.py @@ -35,7 +35,7 @@ def test_set_reserved_launcher_args(): pytest.param("set_verbose_launch", (True,),None,"verbose",id="set_walltime"), ], ) -def test_update_env_initialized(function, value, flag, result): +def test_slurm_class_methods(function, value, flag, result): slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) @@ -75,6 +75,36 @@ def test_format_env_vars(): assert "LOGGING=verbose" in formatted assert all("SSKEYIN" not in x for x in formatted) +def test_catch_existing_env_var(caplog, monkeypatch): + slurmSettings = LaunchSettings( + launcher=LauncherType.SlurmLauncher, + env_vars={ + "SMARTSIM_TEST_VAR": "B", + }, + ) + monkeypatch.setenv("SMARTSIM_TEST_VAR", "A") + monkeypatch.setenv("SMARTSIM_TEST_CSVAR", "A,B") + caplog.clear() + slurmSettings.format_env_vars() + + msg = f"Variable SMARTSIM_TEST_VAR is set to A in current environment. " + msg += f"If the job is running in an interactive allocation, the value B will not be set. " + msg += "Please consider removing the variable from the environment and re-running the experiment." + + for record in caplog.records: + assert record.levelname == "WARNING" + assert record.message == msg + + caplog.clear() + + env_vars = {"SMARTSIM_TEST_VAR": "B", "SMARTSIM_TEST_CSVAR": "C,D"} + settings = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) + settings.format_comma_sep_env_vars() + + for record in caplog.records: + assert record.levelname == "WARNING" + assert record.message == msg + def test_format_comma_sep_env_vars(): """Test format_comma_sep_env_vars runs correctly""" env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} @@ -86,7 +116,7 @@ def test_format_comma_sep_env_vars(): assert "name_0,name_1" not in formatted assert "SSKEYIN=name_0,name_1" in comma_separated_formatted -def test_srun_settings(): +def test_slurmSettings_settings(): """Test format_launcher_args runs correctly""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) slurmLauncher.set_nodes(5) @@ -97,7 +127,7 @@ def test_srun_settings(): result = ["--nodes=5", "--cpus-per-task=2", "--ntasks=100", "--ntasks-per-node=20"] assert formatted == result -def test_srun_launcher_args(): +def test_slurmSettings_launcher_args(): """Test the possible user overrides through run_args""" launcher_args = { "account": "A3123", From 799b0209b7002b1c2f02fe7f4f01aad958613fe2 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 20 May 2024 17:54:19 -0500 Subject: [PATCH 17/43] remove unused import --- .../settings_test/test_lsfScheduler.py | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/tests/temp_tests/settings_test/test_lsfScheduler.py b/tests/temp_tests/settings_test/test_lsfScheduler.py index 6154bdef4a..a5afaf72bc 100644 --- a/tests/temp_tests/settings_test/test_lsfScheduler.py +++ b/tests/temp_tests/settings_test/test_lsfScheduler.py @@ -1,5 +1,4 @@ from smartsim.settingshold import BatchSettings -from smartsim.settingshold.translators.batch.lsf import BsubBatchArgTranslator import pytest import logging from smartsim.settingshold.batchCommand import SchedulerType @@ -37,39 +36,38 @@ def test_create_bsub(): args = lsfScheduler.format_batch_args() assert args == ["-core_isolation", "-nnodes 1", "-W 10:10", "-q default"] -# @pytest.mark.parametrize( -# "method,params", -# [ -# pytest.param("set_tasks", (3,), id="set_tasks"), -# pytest.param("set_smts", (10,), id="set_smts"), -# pytest.param("set_ncpus", (2,), id="set_ncpus"), -# pytest.param("set_project", ("project",), id="set_project"), -# ], -# ) -# def test_unimplimented_setters_throw_warning(caplog, method, params): -# from smartsim.settings.base import logger +@pytest.mark.parametrize( + "method,params", + [ + pytest.param("set_partition", (3,), id="set_tasks"), + pytest.param("set_cpus_per_task", (10,), id="set_smts"), + pytest.param("set_ncpus", (2,), id="set_ncpus"), + ], +) +def test_unimplimented_setters_throw_warning(caplog, method, params): + from smartsim.settings.base import logger -# prev_prop = logger.propagate -# logger.propagate = True + prev_prop = logger.propagate + logger.propagate = True -# with caplog.at_level(logging.WARNING): -# caplog.clear() -# slurmScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) -# try: -# getattr(slurmScheduler, method)(*params) -# finally: -# logger.propagate = prev_prop + with caplog.at_level(logging.WARNING): + caplog.clear() + slurmScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) + try: + getattr(slurmScheduler, method)(*params) + finally: + logger.propagate = prev_prop -# for rec in caplog.records: -# if ( -# logging.WARNING <= rec.levelno < logging.ERROR -# and (method and "not supported" and "bsub") in rec.msg -# ): -# break -# else: -# pytest.fail( -# ( -# f"No message stating method `{method}` is not " -# "implemented at `warning` level" -# ) -# ) \ No newline at end of file + for rec in caplog.records: + if ( + logging.WARNING <= rec.levelno < logging.ERROR + and (method and "not supported" and "bsub") in rec.msg + ): + break + else: + pytest.fail( + ( + f"No message stating method `{method}` is not " + "implemented at `warning` level" + ) + ) \ No newline at end of file From 053eac0d4ae9f1aec6ecadbc87652dae5e5b0f0e Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 23 May 2024 16:33:13 -0500 Subject: [PATCH 18/43] commands --- smartsim/_core/command/__init__.py | 27 +++++++++ smartsim/_core/command/command.py | 60 +++++++++++++++++++ smartsim/_core/command/commandList.py | 25 ++++++++ smartsim/settingshold/batchCommand.py | 26 ++++++++ smartsim/settingshold/batchSettings.py | 26 ++++++++ smartsim/settingshold/common.py | 26 ++++++++ smartsim/settingshold/launchCommand.py | 26 ++++++++ smartsim/settingshold/launchSettings.py | 26 ++++++++ smartsim/settingshold/translators/__init__.py | 26 ++++++++ .../translators/batch/__init.__.py | 26 ++++++++ .../settingshold/translators/batch/lsf.py | 26 ++++++++ .../settingshold/translators/batch/pbs.py | 26 ++++++++ .../settingshold/translators/batch/slurm.py | 26 ++++++++ .../translators/batchArgTranslator.py | 26 ++++++++ .../settingshold/translators/launch/alps.py | 26 ++++++++ .../settingshold/translators/launch/dragon.py | 26 ++++++++ .../settingshold/translators/launch/local.py | 26 ++++++++ .../settingshold/translators/launch/lsf.py | 26 ++++++++ .../settingshold/translators/launch/mpi.py | 26 ++++++++ .../settingshold/translators/launch/pals.py | 26 ++++++++ .../settingshold/translators/launch/slurm.py | 26 ++++++++ .../translators/launchArgTranslator.py | 26 ++++++++ tests/temp_tests/ensemble_tests.py | 11 ---- tests/temp_tests/model_tests.py | 36 ----------- .../test_core/test_commands/test_command.py | 13 ++++ .../test_alpsLauncher.py | 0 .../test_batchSettings.py | 0 .../test_dragonLauncher.py | 0 .../test_launchSettings.py | 0 .../test_localLauncher.py | 0 .../test_lsfLauncher.py | 0 .../test_lsfScheduler.py | 0 .../test_mpiLauncher.py | 0 .../test_palsLauncher.py | 0 .../test_pbsScheduler.py | 0 .../test_slurmLauncher.py | 0 .../test_slurmScheduler.py | 0 37 files changed, 619 insertions(+), 47 deletions(-) create mode 100644 smartsim/_core/command/__init__.py create mode 100644 smartsim/_core/command/command.py create mode 100644 smartsim/_core/command/commandList.py delete mode 100644 tests/temp_tests/ensemble_tests.py delete mode 100644 tests/temp_tests/model_tests.py create mode 100644 tests/temp_tests/test_core/test_commands/test_command.py rename tests/temp_tests/{settings_test => test_settings}/test_alpsLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_batchSettings.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_dragonLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_launchSettings.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_localLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_lsfLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_lsfScheduler.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_mpiLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_palsLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_pbsScheduler.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_slurmLauncher.py (100%) rename tests/temp_tests/{settings_test => test_settings}/test_slurmScheduler.py (100%) diff --git a/smartsim/_core/command/__init__.py b/smartsim/_core/command/__init__.py new file mode 100644 index 0000000000..881e4becb4 --- /dev/null +++ b/smartsim/_core/command/__init__.py @@ -0,0 +1,27 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .command import Command \ No newline at end of file diff --git a/smartsim/_core/command/command.py b/smartsim/_core/command/command.py new file mode 100644 index 0000000000..9af5956860 --- /dev/null +++ b/smartsim/_core/command/command.py @@ -0,0 +1,60 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from collections.abc import MutableSequence +from ...settingshold.launchCommand import LauncherType +import typing as t + +class Command(MutableSequence): + """Basic container for command information + """ + + def __init__(self, launcher: LauncherType, command:t.List[str]) -> None: + self._launcher = launcher + self._command = command + + @property + def launcher(self) -> LauncherType: + return self._launcher + + @property + def command(self) -> t.List[str]: + return self._command + + def __getitem__(self, idx: int) -> str: + return self._command[idx] + + def __setitem__(self, idx: int, value: str) -> None: + self._command[idx] = value + + def __delitem__(self, idx: int) -> None: + del self._command[idx] + + def __len__(self) -> int: + return len(self._command) + + def insert(self, idx: int, value: str) -> None: + self._command.insert(idx, value) \ No newline at end of file diff --git a/smartsim/_core/command/commandList.py b/smartsim/_core/command/commandList.py new file mode 100644 index 0000000000..9852bc0fff --- /dev/null +++ b/smartsim/_core/command/commandList.py @@ -0,0 +1,25 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/smartsim/settingshold/batchCommand.py b/smartsim/settingshold/batchCommand.py index 4cc04df6bf..b92d668111 100644 --- a/smartsim/settingshold/batchCommand.py +++ b/smartsim/settingshold/batchCommand.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from enum import Enum class SchedulerType(Enum): diff --git a/smartsim/settingshold/batchSettings.py b/smartsim/settingshold/batchSettings.py index c55e5cfb29..cce2f01770 100644 --- a/smartsim/settingshold/batchSettings.py +++ b/smartsim/settingshold/batchSettings.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t import copy diff --git a/smartsim/settingshold/common.py b/smartsim/settingshold/common.py index 345c57fa5f..0372772590 100644 --- a/smartsim/settingshold/common.py +++ b/smartsim/settingshold/common.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import typing as t IntegerArgument = t.Dict[str, t.Optional[int]] diff --git a/smartsim/settingshold/launchCommand.py b/smartsim/settingshold/launchCommand.py index a3e346e9ba..fa004677a3 100644 --- a/smartsim/settingshold/launchCommand.py +++ b/smartsim/settingshold/launchCommand.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from enum import Enum class LauncherType(Enum): diff --git a/smartsim/settingshold/launchSettings.py b/smartsim/settingshold/launchSettings.py index 1347c5e2da..f85cc16878 100644 --- a/smartsim/settingshold/launchSettings.py +++ b/smartsim/settingshold/launchSettings.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t import copy diff --git a/smartsim/settingshold/translators/__init__.py b/smartsim/settingshold/translators/__init__.py index af6351065f..2831c0cb1c 100644 --- a/smartsim/settingshold/translators/__init__.py +++ b/smartsim/settingshold/translators/__init__.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from .launchArgTranslator import LaunchArgTranslator from .batchArgTranslator import BatchArgTranslator diff --git a/smartsim/settingshold/translators/batch/__init.__.py b/smartsim/settingshold/translators/batch/__init.__.py index c63e374691..d3478e91fc 100644 --- a/smartsim/settingshold/translators/batch/__init.__.py +++ b/smartsim/settingshold/translators/batch/__init.__.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from .lsf import BsubBatchArgTranslator from .pbs import QsubBatchArgTranslator from .slurm import SlurmBatchArgTranslator diff --git a/smartsim/settingshold/translators/batch/lsf.py b/smartsim/settingshold/translators/batch/lsf.py index 5dc18da135..6a39e5227e 100644 --- a/smartsim/settingshold/translators/batch/lsf.py +++ b/smartsim/settingshold/translators/batch/lsf.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/batch/pbs.py b/smartsim/settingshold/translators/batch/pbs.py index 716f05caff..a65f275d59 100644 --- a/smartsim/settingshold/translators/batch/pbs.py +++ b/smartsim/settingshold/translators/batch/pbs.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations from copy import deepcopy diff --git a/smartsim/settingshold/translators/batch/slurm.py b/smartsim/settingshold/translators/batch/slurm.py index f7cafacd16..fc97e246ab 100644 --- a/smartsim/settingshold/translators/batch/slurm.py +++ b/smartsim/settingshold/translators/batch/slurm.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import re diff --git a/smartsim/settingshold/translators/batchArgTranslator.py b/smartsim/settingshold/translators/batchArgTranslator.py index f32ed423cc..a34ef93399 100644 --- a/smartsim/settingshold/translators/batchArgTranslator.py +++ b/smartsim/settingshold/translators/batchArgTranslator.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations from abc import ABC, abstractmethod diff --git a/smartsim/settingshold/translators/launch/alps.py b/smartsim/settingshold/translators/launch/alps.py index 803aacb897..6d39c2317f 100644 --- a/smartsim/settingshold/translators/launch/alps.py +++ b/smartsim/settingshold/translators/launch/alps.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations from ..launchArgTranslator import LaunchArgTranslator import typing as t diff --git a/smartsim/settingshold/translators/launch/dragon.py b/smartsim/settingshold/translators/launch/dragon.py index fdf027f1a3..d98372b894 100644 --- a/smartsim/settingshold/translators/launch/dragon.py +++ b/smartsim/settingshold/translators/launch/dragon.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/launch/local.py b/smartsim/settingshold/translators/launch/local.py index c1124ceb2b..72bc60134c 100644 --- a/smartsim/settingshold/translators/launch/local.py +++ b/smartsim/settingshold/translators/launch/local.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/launch/lsf.py b/smartsim/settingshold/translators/launch/lsf.py index 7e7469c5d9..ae95f0961c 100644 --- a/smartsim/settingshold/translators/launch/lsf.py +++ b/smartsim/settingshold/translators/launch/lsf.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/launch/mpi.py b/smartsim/settingshold/translators/launch/mpi.py index 1097d758ce..2b1a932c08 100644 --- a/smartsim/settingshold/translators/launch/mpi.py +++ b/smartsim/settingshold/translators/launch/mpi.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/launch/pals.py b/smartsim/settingshold/translators/launch/pals.py index ca5afb0550..e59f70870f 100644 --- a/smartsim/settingshold/translators/launch/pals.py +++ b/smartsim/settingshold/translators/launch/pals.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/launch/slurm.py b/smartsim/settingshold/translators/launch/slurm.py index 685b8a6631..e1a9b2d719 100644 --- a/smartsim/settingshold/translators/launch/slurm.py +++ b/smartsim/settingshold/translators/launch/slurm.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations import typing as t diff --git a/smartsim/settingshold/translators/launchArgTranslator.py b/smartsim/settingshold/translators/launchArgTranslator.py index b02a43a58b..ea3d18d7d2 100644 --- a/smartsim/settingshold/translators/launchArgTranslator.py +++ b/smartsim/settingshold/translators/launchArgTranslator.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from __future__ import annotations from abc import ABC, abstractmethod diff --git a/tests/temp_tests/ensemble_tests.py b/tests/temp_tests/ensemble_tests.py deleted file mode 100644 index b0c98fe7c5..0000000000 --- a/tests/temp_tests/ensemble_tests.py +++ /dev/null @@ -1,11 +0,0 @@ -from smartsim.entity import Ensemble -from smartsim.settings import RunSettings - -def test_create_ensemble(): - run_settings = RunSettings() - ensemble = Ensemble(name="model", exe="echo", run_settings=run_settings, exe_args=["hello"], replicas=2) - assert ensemble.exe == "echo" - assert ensemble.exe_args == ["hello"] - for model in ensemble: - assert model.exe == ['/usr/bin/echo'] - assert model.exe_args == ["hello"] \ No newline at end of file diff --git a/tests/temp_tests/model_tests.py b/tests/temp_tests/model_tests.py deleted file mode 100644 index 37ec521020..0000000000 --- a/tests/temp_tests/model_tests.py +++ /dev/null @@ -1,36 +0,0 @@ -from smartsim.entity import Ensemble, Model -from smartsim.settings import RunSettings, SrunSettings -from smartsim.database import Orchestrator -from smartsim import Experiment -from smartsim.status import SmartSimStatus - -def test_model_constructor(): - run_settings = RunSettings() - model = Model(name="testing", run_settings=run_settings, exe="echo", exe_args=["hello"], params={}) - assert model.exe == ['/usr/bin/echo'] - assert model.exe_args == ["hello"] - -def test_model_add_exe_args(): - run_settings = SrunSettings() - model = Model(name="testing", run_settings=run_settings, exe="echo", exe_args=["hello"], params={}) - model.add_exe_args("there") - assert model.exe_args == ["hello", "there"] - model.add_exe_args(["how", "are", "you"]) - assert model.exe_args == ["hello", "there", "how", "are", "you"] - -def test_create_model(): - run_settings = SrunSettings() - exp = Experiment("exp") - model = exp.create_model(name="model", run_settings=run_settings, exe="echo", exe_args=["hello"]) - assert model.exe == ['/usr/bin/echo'] - assert model.exe_args == ["hello"] - -def test_start_a_model(): - exp = Experiment("exp") - run_settings = SrunSettings() - model = Model(name="testing", exe="echo", run_settings=run_settings, exe_args=["hello"], params={}) - assert model.exe == ['/usr/bin/echo'] - assert model.exe_args == ["hello"] - exp.start(model) - model_status = exp.get_status(model)[0] - assert model_status != SmartSimStatus.STATUS_FAILED \ No newline at end of file diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py new file mode 100644 index 0000000000..4df4381c9b --- /dev/null +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -0,0 +1,13 @@ +import pytest +from .....smartsim._core.command import Command +from smartsim.settingshold.launchCommand import LauncherType + +def test_command_initialization_with_valid_inputs(): + cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + assert cmd.command == ["salloc", "-N", "1"] + assert cmd.launcher == LauncherType.SlurmLauncher + +# def test_command_initialization_with_empty_command_list(): + +# def test_command_initialization_with_invalid_launcher(): + \ No newline at end of file diff --git a/tests/temp_tests/settings_test/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_alpsLauncher.py rename to tests/temp_tests/test_settings/test_alpsLauncher.py diff --git a/tests/temp_tests/settings_test/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py similarity index 100% rename from tests/temp_tests/settings_test/test_batchSettings.py rename to tests/temp_tests/test_settings/test_batchSettings.py diff --git a/tests/temp_tests/settings_test/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_dragonLauncher.py rename to tests/temp_tests/test_settings/test_dragonLauncher.py diff --git a/tests/temp_tests/settings_test/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py similarity index 100% rename from tests/temp_tests/settings_test/test_launchSettings.py rename to tests/temp_tests/test_settings/test_launchSettings.py diff --git a/tests/temp_tests/settings_test/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_localLauncher.py rename to tests/temp_tests/test_settings/test_localLauncher.py diff --git a/tests/temp_tests/settings_test/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_lsfLauncher.py rename to tests/temp_tests/test_settings/test_lsfLauncher.py diff --git a/tests/temp_tests/settings_test/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py similarity index 100% rename from tests/temp_tests/settings_test/test_lsfScheduler.py rename to tests/temp_tests/test_settings/test_lsfScheduler.py diff --git a/tests/temp_tests/settings_test/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_mpiLauncher.py rename to tests/temp_tests/test_settings/test_mpiLauncher.py diff --git a/tests/temp_tests/settings_test/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_palsLauncher.py rename to tests/temp_tests/test_settings/test_palsLauncher.py diff --git a/tests/temp_tests/settings_test/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py similarity index 100% rename from tests/temp_tests/settings_test/test_pbsScheduler.py rename to tests/temp_tests/test_settings/test_pbsScheduler.py diff --git a/tests/temp_tests/settings_test/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py similarity index 100% rename from tests/temp_tests/settings_test/test_slurmLauncher.py rename to tests/temp_tests/test_settings/test_slurmLauncher.py diff --git a/tests/temp_tests/settings_test/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py similarity index 100% rename from tests/temp_tests/settings_test/test_slurmScheduler.py rename to tests/temp_tests/test_settings/test_slurmScheduler.py From 1681848c1ff4f7ae06e633c66c764d34b4280af9 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 23 May 2024 17:39:19 -0500 Subject: [PATCH 19/43] command implementation --- .../_core/{command => commands}/__init__.py | 0 .../_core/{command => commands}/command.py | 15 ++++++++- .../{command => commands}/commandList.py | 0 .../test_core/test_commands/test_command.py | 33 +++++++++++++++---- 4 files changed, 41 insertions(+), 7 deletions(-) rename smartsim/_core/{command => commands}/__init__.py (100%) rename smartsim/_core/{command => commands}/command.py (85%) rename smartsim/_core/{command => commands}/commandList.py (100%) diff --git a/smartsim/_core/command/__init__.py b/smartsim/_core/commands/__init__.py similarity index 100% rename from smartsim/_core/command/__init__.py rename to smartsim/_core/commands/__init__.py diff --git a/smartsim/_core/command/command.py b/smartsim/_core/commands/command.py similarity index 85% rename from smartsim/_core/command/command.py rename to smartsim/_core/commands/command.py index 9af5956860..38f4f250b9 100644 --- a/smartsim/_core/command/command.py +++ b/smartsim/_core/commands/command.py @@ -31,30 +31,43 @@ class Command(MutableSequence): """Basic container for command information """ - def __init__(self, launcher: LauncherType, command:t.List[str]) -> None: self._launcher = launcher self._command = command @property def launcher(self) -> LauncherType: + """Get the launcher type. + """ return self._launcher @property def command(self) -> t.List[str]: + """Get the command list. + """ return self._command def __getitem__(self, idx: int) -> str: + """Get the command at the specified index. + """ return self._command[idx] def __setitem__(self, idx: int, value: str) -> None: + """Set the command at the specified index. + """ self._command[idx] = value def __delitem__(self, idx: int) -> None: + """Delete the command at the specified index. + """ del self._command[idx] def __len__(self) -> int: + """Get the length of the command list. + """ return len(self._command) def insert(self, idx: int, value: str) -> None: + """Insert a command at the specified index. + """ self._command.insert(idx, value) \ No newline at end of file diff --git a/smartsim/_core/command/commandList.py b/smartsim/_core/commands/commandList.py similarity index 100% rename from smartsim/_core/command/commandList.py rename to smartsim/_core/commands/commandList.py diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 4df4381c9b..2456578a25 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -1,13 +1,34 @@ import pytest -from .....smartsim._core.command import Command +from smartsim._core.commands.command import Command from smartsim.settingshold.launchCommand import LauncherType -def test_command_initialization_with_valid_inputs(): +def test_command_init(): cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] assert cmd.launcher == LauncherType.SlurmLauncher -# def test_command_initialization_with_empty_command_list(): - -# def test_command_initialization_with_invalid_launcher(): - \ No newline at end of file +def test_command_getitem(): + cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + get_value = cmd[0] + assert get_value == "salloc" + +def test_command_setitem(): + cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + cmd[0] = "srun" + cmd[1] = "-n" + assert cmd.command == ["srun", "-n", "1"] + +def test_command_delitem(): + cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1", "--constraint", "P100"]) + del(cmd.command[3]) + del(cmd.command[3]) + assert cmd.command == ["salloc", "-N", "1"] + +def test_command_len(): + cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + assert len(cmd) is 3 + +def test_command_insert(): + cmd = Command(launcher=LauncherType.SlurmLauncher, command=["-N", "1"]) + cmd.insert(0, "salloc") + assert cmd.command == ["salloc", "-N", "1"] \ No newline at end of file From acf37e39d14484927f72ef396d90d3478a24f681 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 23 May 2024 17:58:21 -0500 Subject: [PATCH 20/43] commandList added --- smartsim/_core/commands/__init__.py | 3 +- smartsim/_core/commands/command.py | 2 + smartsim/_core/commands/commandList.py | 45 ++++++++++++++++++- .../test_commands/test_commandList.py | 36 +++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 tests/temp_tests/test_core/test_commands/test_commandList.py diff --git a/smartsim/_core/commands/__init__.py b/smartsim/_core/commands/__init__.py index 881e4becb4..670e3e06b5 100644 --- a/smartsim/_core/commands/__init__.py +++ b/smartsim/_core/commands/__init__.py @@ -24,4 +24,5 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .command import Command \ No newline at end of file +from .command import Command +from .commandList import CommandList \ No newline at end of file diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 38f4f250b9..c5533375a1 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -32,6 +32,8 @@ class Command(MutableSequence): """Basic container for command information """ def __init__(self, launcher: LauncherType, command:t.List[str]) -> None: + """Command constructor + """ self._launcher = launcher self._command = command diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 9852bc0fff..9c4abceb7a 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -22,4 +22,47 @@ # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from collections.abc import MutableSequence +from .command import Command +import typing as t + +class CommandList(MutableSequence): + """Container for a seuence of commands + """ + def __init__(self, commands: t.Optional[t.Union[Command, t.List[Command]]]): + """CommandList constructor + """ + self._commands: t.List[Command] = list(commands) + + @property + def commands(self) -> t.List[Command]: + """Get the command list. + """ + return self._commands + + def __getitem__(self, idx: int) -> Command: + """Get the command at the specified index. + """ + return self._commands[idx] + + def __setitem__(self, idx: int, value: Command) -> None: + """Set the command at the specified index. + """ + self._commands[idx] = value + + def __delitem__(self, idx: int) -> None: + """Delete the command at the specified index. + """ + del self._commands[idx] + + def __len__(self) -> int: + """Get the length of the command list. + """ + return len(self._commands) + + def insert(self, idx: int, value: Command) -> None: + """Insert a command at the specified index. + """ + self._commands.insert(idx, value) \ No newline at end of file diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py new file mode 100644 index 0000000000..ba9db2a703 --- /dev/null +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -0,0 +1,36 @@ +import pytest +from smartsim._core.commands.commandList import CommandList +from smartsim._core.commands.command import Command +from smartsim.settingshold.launchCommand import LauncherType + +salloc_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) +srun_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["srun", "-n", "1"]) +sacct_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["sacct", "--user"]) + +def test_command_init(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + assert cmd_list.commands == [salloc_cmd,srun_cmd] + +def test_command_getitem(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + get_value = cmd_list[0] + assert get_value == salloc_cmd + +def test_command_setitem(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + cmd_list[0] = sacct_cmd + assert cmd_list.commands == [sacct_cmd,srun_cmd] + +def test_command_delitem(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + del(cmd_list.commands[0]) + assert cmd_list.commands == [srun_cmd] + +def test_command_len(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + assert len(cmd_list) is 2 + +def test_command_insert(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + cmd_list.insert(0, sacct_cmd) + assert cmd_list.commands == [sacct_cmd,salloc_cmd,srun_cmd] \ No newline at end of file From 2f48f5a005b71bcad8857904dfab358f33987b30 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 23 May 2024 18:30:29 -0500 Subject: [PATCH 21/43] str dunder --- smartsim/_core/commands/command.py | 8 ++++++- smartsim/_core/commands/commandList.py | 23 ++++++++++++------- .../test_commands/test_commandList.py | 6 ++++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index c5533375a1..69744e9428 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -72,4 +72,10 @@ def __len__(self) -> int: def insert(self, idx: int, value: str) -> None: """Insert a command at the specified index. """ - self._command.insert(idx, value) \ No newline at end of file + self._command.insert(idx, value) + + def __str__(self) -> str: # pragma: no cover + string = "" + string += f"\nLauncher: {self.launcher.value}\n" + string += f"Command: {' '.join(str(cmd) for cmd in self.command)}" + return string \ No newline at end of file diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 9c4abceb7a..886a49004d 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -29,7 +29,7 @@ import typing as t class CommandList(MutableSequence): - """Container for a seuence of commands + """Container for a Sequence of Command objects """ def __init__(self, commands: t.Optional[t.Union[Command, t.List[Command]]]): """CommandList constructor @@ -38,31 +38,38 @@ def __init__(self, commands: t.Optional[t.Union[Command, t.List[Command]]]): @property def commands(self) -> t.List[Command]: - """Get the command list. + """Get the Command list. """ return self._commands def __getitem__(self, idx: int) -> Command: - """Get the command at the specified index. + """Get the Command at the specified index. """ return self._commands[idx] def __setitem__(self, idx: int, value: Command) -> None: - """Set the command at the specified index. + """Set the Command at the specified index. """ self._commands[idx] = value def __delitem__(self, idx: int) -> None: - """Delete the command at the specified index. + """Delete the Command at the specified index. """ del self._commands[idx] def __len__(self) -> int: - """Get the length of the command list. + """Get the length of the Command list. """ return len(self._commands) def insert(self, idx: int, value: Command) -> None: - """Insert a command at the specified index. + """Insert a Command at the specified index. """ - self._commands.insert(idx, value) \ No newline at end of file + self._commands.insert(idx, value) + + def __str__(self) -> str: # pragma: no cover + string = "\n\nCommand List:\n\n" + for counter, cmd in enumerate(self.commands): + string += f"CommandList index {counter} value:" + string += f"{cmd}\n\n" + return string \ No newline at end of file diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index ba9db2a703..b0052223c6 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -33,4 +33,8 @@ def test_command_len(): def test_command_insert(): cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) cmd_list.insert(0, sacct_cmd) - assert cmd_list.commands == [sacct_cmd,salloc_cmd,srun_cmd] \ No newline at end of file + assert cmd_list.commands == [sacct_cmd,salloc_cmd,srun_cmd] + +def test_command_str(): + cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + print(cmd_list) \ No newline at end of file From 119903e6fa5079fb35a2835f5aba0d73941b9969 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 23 May 2024 18:32:02 -0500 Subject: [PATCH 22/43] remove unused imports --- tests/temp_tests/test_core/test_commands/test_command.py | 1 - .../temp_tests/test_core/test_commands/test_commandList.py | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 2456578a25..f1f1650265 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -1,4 +1,3 @@ -import pytest from smartsim._core.commands.command import Command from smartsim.settingshold.launchCommand import LauncherType diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index b0052223c6..3a2c5b1779 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -1,4 +1,3 @@ -import pytest from smartsim._core.commands.commandList import CommandList from smartsim._core.commands.command import Command from smartsim.settingshold.launchCommand import LauncherType @@ -33,8 +32,4 @@ def test_command_len(): def test_command_insert(): cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) cmd_list.insert(0, sacct_cmd) - assert cmd_list.commands == [sacct_cmd,salloc_cmd,srun_cmd] - -def test_command_str(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) - print(cmd_list) \ No newline at end of file + assert cmd_list.commands == [sacct_cmd,salloc_cmd,srun_cmd] \ No newline at end of file From 804786a4decc85b99f83c490937154673d24889c Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 23 May 2024 18:32:51 -0500 Subject: [PATCH 23/43] copyright --- .../test_core/test_commands/test_command.py | 26 +++++++++++++++++++ .../test_commands/test_commandList.py | 26 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index f1f1650265..61b748c0fa 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from smartsim._core.commands.command import Command from smartsim.settingshold.launchCommand import LauncherType diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 3a2c5b1779..6b62dcee93 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -1,3 +1,29 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from smartsim._core.commands.commandList import CommandList from smartsim._core.commands.command import Command from smartsim.settingshold.launchCommand import LauncherType From faf2d15f6ec9dbb5958d6ed07598cb28c83959ef Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Mon, 27 May 2024 19:14:24 -0500 Subject: [PATCH 24/43] move tests --- smartsim/settings/base.py | 629 ------------------ smartsim/settings/settings.py | 196 ------ testing_1/temp_tests/steps_tests.py | 110 --- tests/temp_tests/steps_tests.py | 71 +- .../test_core/test_commands/test_command.py | 0 .../test_commands/test_commandList.py | 0 .../test_settings/test_alpsLauncher.py | 0 .../test_settings/test_batchSettings.py | 0 .../test_settings/test_dragonLauncher.py | 0 .../test_settings/test_launchSettings.py | 0 .../test_settings/test_localLauncher.py | 0 .../test_settings/test_lsfLauncher.py | 0 .../test_settings/test_lsfScheduler.py | 0 .../test_settings/test_mpiLauncher.py | 0 .../test_settings/test_palsLauncher.py | 0 .../test_settings/test_pbsScheduler.py | 0 .../test_settings/test_slurmLauncher.py | 0 .../test_settings/test_slurmScheduler.py | 0 18 files changed, 21 insertions(+), 985 deletions(-) delete mode 100644 smartsim/settings/base.py delete mode 100644 smartsim/settings/settings.py delete mode 100644 testing_1/temp_tests/steps_tests.py rename {testing_1 => tests}/temp_tests/test_core/test_commands/test_command.py (100%) rename {testing_1 => tests}/temp_tests/test_core/test_commands/test_commandList.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_alpsLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_batchSettings.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_dragonLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_launchSettings.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_localLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_lsfLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_lsfScheduler.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_mpiLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_palsLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_pbsScheduler.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_slurmLauncher.py (100%) rename {testing_1 => tests}/temp_tests/test_settings/test_slurmScheduler.py (100%) diff --git a/smartsim/settings/base.py b/smartsim/settings/base.py deleted file mode 100644 index 57036f91c0..0000000000 --- a/smartsim/settings/base.py +++ /dev/null @@ -1,629 +0,0 @@ -# BSD 2-Clause License # -# Copyright (c) 2021-2024, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import annotations - -import copy -import typing as t - -from smartsim.settings.containers import Container - -from .._core.utils.helpers import expand_exe_path, fmt_dict, is_valid_cmd -from ..entity.dbobject import DBModel, DBScript -from ..log import get_logger - -logger = get_logger(__name__) - -# fmt: off -class SettingsBase: - ... -# fmt: on - - -# pylint: disable=too-many-public-methods -class RunSettings(SettingsBase): - # pylint: disable=unused-argument - - def __init__( - self, - run_command: str = "", - run_args: t.Optional[t.Dict[str, t.Union[int, str, float, None]]] = None, - env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, - container: t.Optional[Container] = None, - **_kwargs: t.Any, - ) -> None: - """Run parameters for a ``Model`` - - The base ``RunSettings`` class should only be used with the `local` - launcher on single node, workstations, or laptops. - - If no ``run_command`` is specified, the executable will be launched - locally. - - ``run_args`` passed as a dict will be interpreted literally for - local ``RunSettings`` and added directly to the ``run_command`` - e.g. run_args = {"-np": 2} will be "-np 2" - - Example initialization - - .. highlight:: python - .. code-block:: python - - rs = RunSettings("echo", "hello", "mpirun", run_args={"-np": "2"}) - - :param run_command: launch binary (e.g. "srun") - :param run_args: arguments for run command (e.g. `-np` for `mpiexec`) - :param env_vars: environment vars to launch job with - :param container: container type for workload (e.g. "singularity") - """ - # Do not expand executable if running within a container - self.run_args = run_args or {} - self.env_vars = env_vars or {} - self.container = container - self._run_command = run_command - self.in_batch = False - self.colocated_db_settings: t.Optional[ - t.Dict[ - str, - t.Union[ - bool, - int, - str, - None, - t.List[str], - t.Iterable[t.Union[int, t.Iterable[int]]], - t.List[DBModel], - t.List[DBScript], - t.Dict[str, t.Union[int, None]], - t.Dict[str, str], - ], - ] - ] = None - - @property - def run_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: - """Return an immutable list of attached run arguments. - - :returns: attached run arguments - """ - return self._run_args - - @run_args.setter - def run_args(self, value: t.Dict[str, t.Union[int, str, float, None]]) -> None: - """Set the run arguments. - - :param value: run arguments - """ - self._run_args = copy.deepcopy(value) - - @property - def env_vars(self) -> t.Dict[str, t.Optional[str]]: - """Return an immutable list of attached environment variables. - - :returns: attached environment variables - """ - return self._env_vars - - @env_vars.setter - def env_vars(self, value: t.Dict[str, t.Optional[str]]) -> None: - """Set the environment variables. - - :param value: environment variables - """ - self._env_vars = copy.deepcopy(value) - - # To be overwritten by subclasses. Set of reserved args a user cannot change - reserved_run_args = set() # type: set[str] - - def set_nodes(self, nodes: int) -> None: - """Set the number of nodes - - :param nodes: number of nodes to run with - """ - logger.warning( - ( - "Node specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_tasks(self, tasks: int) -> None: - """Set the number of tasks to launch - - :param tasks: number of tasks to launch - """ - logger.warning( - ( - "Task specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_tasks_per_node(self, tasks_per_node: int) -> None: - """Set the number of tasks per node - - :param tasks_per_node: number of tasks to launch per node - """ - logger.warning( - ( - "Task per node specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_task_map(self, task_mapping: str) -> None: - """Set a task mapping - - :param task_mapping: task mapping - """ - logger.warning( - ( - "Task mapping specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_cpus_per_task(self, cpus_per_task: int) -> None: - """Set the number of cpus per task - - :param cpus_per_task: number of cpus per task - """ - logger.warning( - ( - "CPU per node specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - """Specify the hostlist for this job - - :param host_list: hosts to launch on - """ - logger.warning( - ( - "Hostlist specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_hostlist_from_file(self, file_path: str) -> None: - """Use the contents of a file to specify the hostlist for this job - - :param file_path: Path to the hostlist file - """ - logger.warning( - ( - "Hostlist from file specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: - """Specify a list of hosts to exclude for launching this job - - :param host_list: hosts to exclude - """ - logger.warning( - ( - "Excluded host list specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: - """Set the cores to which MPI processes are bound - - :param bindings: List specifing the cores to which MPI processes are bound - """ - logger.warning( - ( - "CPU binding specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_memory_per_node(self, memory_per_node: int) -> None: - """Set the amount of memory required per node in megabytes - - :param memory_per_node: Number of megabytes per node - """ - logger.warning( - ( - "Memory per node specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_verbose_launch(self, verbose: bool) -> None: - """Set the job to run in verbose mode - - :param verbose: Whether the job should be run verbosely - """ - logger.warning( - ( - "Verbose specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_quiet_launch(self, quiet: bool) -> None: - """Set the job to run in quiet mode - - :param quiet: Whether the job should be run quietly - """ - logger.warning( - ( - "Quiet specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_broadcast(self, dest_path: t.Optional[str] = None) -> None: - """Copy executable file to allocated compute nodes - - :param dest_path: Path to copy an executable file - """ - logger.warning( - ( - "Broadcast specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_time(self, hours: int = 0, minutes: int = 0, seconds: int = 0) -> None: - """Automatically format and set wall time - - :param hours: number of hours to run job - :param minutes: number of minutes to run job - :param seconds: number of seconds to run job - """ - return self.set_walltime( - self._fmt_walltime(int(hours), int(minutes), int(seconds)) - ) - - def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: - """Specify the node feature for this job - - :param feature_list: node feature to launch on - """ - logger.warning( - ( - "Feature specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - @staticmethod - def _fmt_walltime(hours: int, minutes: int, seconds: int) -> str: - """Convert hours, minutes, and seconds into valid walltime format - - By defualt the formatted wall time is the total number of seconds. - - :param hours: number of hours to run job - :param minutes: number of minutes to run job - :param seconds: number of seconds to run job - :returns: Formatted walltime - """ - time_ = hours * 3600 - time_ += minutes * 60 - time_ += seconds - return str(time_) - - def set_walltime(self, walltime: str) -> None: - """Set the formatted walltime - - :param walltime: Time in format required by launcher`` - """ - logger.warning( - ( - "Walltime specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_binding(self, binding: str) -> None: - """Set binding - - :param binding: Binding - """ - logger.warning( - ( - "binding specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def set_mpmd_preamble(self, preamble_lines: t.List[str]) -> None: - """Set preamble to a file to make a job MPMD - - :param preamble_lines: lines to put at the beginning of a file. - """ - logger.warning( - ( - "MPMD preamble specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - def make_mpmd(self, settings: RunSettings) -> None: - """Make job an MPMD job - - :param settings: ``RunSettings`` instance - """ - logger.warning( - ( - "Make MPMD specification not implemented for this " - f"RunSettings type: {type(self)}" - ) - ) - - @property - def run_command(self) -> t.Optional[str]: - """Return the launch binary used to launch the executable - - Attempt to expand the path to the executable if possible - - :returns: launch binary e.g. mpiexec - """ - cmd = self._run_command - - if cmd: - if is_valid_cmd(cmd): - # command is valid and will be expanded - return expand_exe_path(cmd) - # command is not valid, so return it as is - # it may be on the compute nodes but not local machine - return cmd - # run without run command - return None - - def update_env(self, env_vars: t.Dict[str, t.Union[str, int, float, bool]]) -> None: - """Update the job environment variables - - To fully inherit the current user environment, add the - workload-manager-specific flag to the launch command. For example, - ``--export=ALL`` for slurm, or ``-V`` for PBS/aprun. - - :param env_vars: environment variables to update or add - :raises TypeError: if env_vars values cannot be coerced to strings - """ - val_types = (str, int, float, bool) - # Coerce env_vars values to str as a convenience to user - for env, val in env_vars.items(): - if not isinstance(val, val_types): - raise TypeError( - f"env_vars[{env}] was of type {type(val)}, not {val_types}" - ) - - self.env_vars[env] = str(val) - - def set( - self, arg: str, value: t.Optional[str] = None, condition: bool = True - ) -> None: - """Allows users to set individual run arguments. - - A method that allows users to set run arguments after object - instantiation. Does basic formatting such as stripping leading dashes. - If the argument has been set previously, this method will log warning - but ultimately comply. - - Conditional expressions may be passed to the conditional parameter. If the - expression evaluates to True, the argument will be set. In not an info - message is logged and no further operation is performed. - - Basic Usage - - .. highlight:: python - .. code-block:: python - - rs = RunSettings("python") - rs.set("an-arg", "a-val") - rs.set("a-flag") - rs.format_run_args() # returns ["an-arg", "a-val", "a-flag", "None"] - - Slurm Example with Conditional Setting - - .. highlight:: python - .. code-block:: python - - import socket - - rs = SrunSettings("echo", "hello") - rs.set_tasks(1) - rs.set("exclusive") - - # Only set this argument if condition param evals True - # Otherwise log and NOP - rs.set("partition", "debug", - condition=socket.gethostname()=="testing-system") - - rs.format_run_args() - # returns ["exclusive", "None", "partition", "debug"] iff - socket.gethostname()=="testing-system" - # otherwise returns ["exclusive", "None"] - - :param arg: name of the argument - :param value: value of the argument - :param conditon: set the argument if condition evaluates to True - """ - if not isinstance(arg, str): - raise TypeError("Argument name should be of type str") - if value is not None and not isinstance(value, str): - raise TypeError("Argument value should be of type str or None") - arg = arg.strip().lstrip("-") - - if not condition: - logger.info(f"Could not set argument '{arg}': condition not met") - return - if arg in self.reserved_run_args: - logger.warning( - ( - f"Could not set argument '{arg}': " - f"it is a reserved arguement of '{type(self).__name__}'" - ) - ) - return - - if arg in self.run_args and value != self.run_args[arg]: - logger.warning(f"Overwritting argument '{arg}' with value '{value}'") - self.run_args[arg] = value - - def format_run_args(self) -> t.List[str]: - """Return formatted run arguments - - For ``RunSettings``, the run arguments are passed - literally with no formatting. - - :return: list run arguments for these settings - """ - formatted = [] - for arg, value in self.run_args.items(): - formatted.append(arg) - formatted.append(str(value)) - return formatted - - def format_env_vars(self) -> t.List[str]: - """Build environment variable string - - :returns: formatted list of strings to export variables - """ - formatted = [] - for key, val in self.env_vars.items(): - if val is None: - formatted.append(f"{key}=") - else: - formatted.append(f"{key}={val}") - return formatted - - def __str__(self) -> str: # pragma: no-cover - string = "" - if self.run_command: - string += f"\nRun Command: {self.run_command}" - if self.run_args: - string += f"\nRun Arguments:\n{fmt_dict(self.run_args)}" - if self.colocated_db_settings: - string += "\nCo-located Database: True" - return string - - -class BatchSettings(SettingsBase): - def __init__( - self, - batch_cmd: str, - batch_args: t.Optional[t.Dict[str, t.Optional[str]]] = None, - **kwargs: t.Any, - ) -> None: - self._batch_cmd = batch_cmd - self.batch_args = batch_args or {} - self._preamble: t.List[str] = [] - self.set_nodes(kwargs.get("nodes", None)) - self.set_walltime(kwargs.get("time", None)) - self.set_queue(kwargs.get("queue", None)) - self.set_account(kwargs.get("account", None)) - - @property - def batch_cmd(self) -> str: - """Return the batch command - - Tests to see if we can expand the batch command - path. If we can, then returns the expanded batch - command. If we cannot, returns the batch command as is. - - :returns: batch command - """ - if is_valid_cmd(self._batch_cmd): - return expand_exe_path(self._batch_cmd) - - return self._batch_cmd - - @property - def batch_args(self) -> t.Dict[str, t.Optional[str]]: - """Retrieve attached batch arguments - - :returns: attached batch arguments - """ - return self._batch_args - - @batch_args.setter - def batch_args(self, value: t.Dict[str, t.Optional[str]]) -> None: - """Attach batch arguments - - :param value: dictionary of batch arguments - """ - self._batch_args = copy.deepcopy(value) if value else {} - - def set_nodes(self, num_nodes: int) -> None: - raise NotImplementedError - - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - raise NotImplementedError - - def set_queue(self, queue: str) -> None: - raise NotImplementedError - - def set_walltime(self, walltime: str) -> None: - raise NotImplementedError - - def set_account(self, account: str) -> None: - raise NotImplementedError - - def format_batch_args(self) -> t.List[str]: - raise NotImplementedError - - def set_batch_command(self, command: str) -> None: - """Set the command used to launch the batch e.g. ``sbatch`` - - :param command: batch command - """ - self._batch_cmd = command - - def add_preamble(self, lines: t.List[str]) -> None: - """Add lines to the batch file preamble. The lines are just - written (unmodified) at the beginning of the batch file - (after the WLM directives) and can be used to e.g. - start virtual environments before running the executables. - - :param line: lines to add to preamble. - """ - if isinstance(lines, str): - self._preamble += [lines] - elif isinstance(lines, list): - self._preamble += lines - else: - raise TypeError("Expected str or List[str] for lines argument") - - @property - def preamble(self) -> t.Iterable[str]: - """Return an iterable of preamble clauses to be prepended to the batch file - - :return: attached preamble clauses - """ - return (clause for clause in self._preamble) - - def __str__(self) -> str: # pragma: no-cover - string = f"Batch Command: {self._batch_cmd}" - if self.batch_args: - string += f"\nBatch arguments:\n{fmt_dict(self.batch_args)}" - return string diff --git a/smartsim/settings/settings.py b/smartsim/settings/settings.py deleted file mode 100644 index 4df86a1232..0000000000 --- a/smartsim/settings/settings.py +++ /dev/null @@ -1,196 +0,0 @@ -# BSD 2-Clause License -# -# Copyright (c) 2021-2024, Hewlett Packard Enterprise -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import typing as t - -from .._core.utils.helpers import is_valid_cmd -from ..error import SmartSimError -from ..settings import ( - AprunSettings, - BsubBatchSettings, - Container, - DragonRunSettings, - JsrunSettings, - MpiexecSettings, - MpirunSettings, - OrterunSettings, - PalsMpiexecSettings, - QsubBatchSettings, - RunSettings, - SbatchSettings, - SrunSettings, - base, -) -from ..wlm import detect_launcher - -_TRunSettingsSelector = t.Callable[[str], t.Callable[..., RunSettings]] - - -def create_batch_settings( - launcher: str, - nodes: t.Optional[int] = None, - time: str = "", - queue: t.Optional[str] = None, - account: t.Optional[str] = None, - batch_args: t.Optional[t.Dict[str, str]] = None, - **kwargs: t.Any, -) -> base.BatchSettings: - """Create a ``BatchSettings`` instance - - See Experiment.create_batch_settings for details - - :param launcher: launcher for this experiment, if set to 'auto', - an attempt will be made to find an available launcher on the system - :param nodes: number of nodes for batch job - :param time: length of batch job - :param queue: queue or partition (if slurm) - :param account: user account name for batch system - :param batch_args: additional batch arguments - :return: a newly created BatchSettings instance - :raises SmartSimError: if batch creation fails - """ - # all supported batch class implementations - by_launcher: t.Dict[str, t.Callable[..., base.BatchSettings]] = { - "pbs": QsubBatchSettings, - "slurm": SbatchSettings, - "lsf": BsubBatchSettings, - "pals": QsubBatchSettings, - } - - if launcher in ["auto", "dragon"]: - launcher = detect_launcher() - if launcher == "dragon": - by_launcher["dragon"] = by_launcher[launcher] - - if launcher == "local": - raise SmartSimError("Local launcher does not support batch workloads") - - # detect the batch class to use based on the launcher provided by - # the user - try: - batch_class = by_launcher[launcher] - batch_settings = batch_class( - nodes=nodes, - time=time, - batch_args=batch_args, - queue=queue, - account=account, - **kwargs, - ) - return batch_settings - - except KeyError: - raise SmartSimError( - f"User attempted to make batch settings for unsupported launcher {launcher}" - ) from None - - -def create_run_settings( - launcher: str, - run_command: str = "auto", - run_args: t.Optional[t.Dict[str, t.Union[int, str, float, None]]] = None, - env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, - container: t.Optional[Container] = None, - **kwargs: t.Any, -) -> RunSettings: - """Create a ``RunSettings`` instance. - - See Experiment.create_run_settings docstring for more details - - :param launcher: launcher to create settings for, if set to 'auto', - an attempt will be made to find an available launcher on the system - :param run_command: command to run the executable - :param run_args: arguments to pass to the ``run_command`` - :param env_vars: environment variables to pass to the executable - :param container: container type for workload (e.g. "singularity") - :return: the created ``RunSettings`` - :raises SmartSimError: if run_command=="auto" and detection fails - """ - # all supported RunSettings child classes - supported: t.Dict[str, _TRunSettingsSelector] = { - "aprun": lambda launcher: AprunSettings, - "srun": lambda launcher: SrunSettings, - "mpirun": lambda launcher: MpirunSettings, - "mpiexec": lambda launcher: ( - MpiexecSettings if launcher != "pals" else PalsMpiexecSettings - ), - "orterun": lambda launcher: OrterunSettings, - "jsrun": lambda launcher: JsrunSettings, - } - - # run commands supported by each launcher - # in order of suspected user preference - by_launcher = { - "dragon": [""], - "slurm": ["srun", "mpirun", "mpiexec"], - "pbs": ["aprun", "mpirun", "mpiexec"], - "pals": ["mpiexec"], - "lsf": ["jsrun", "mpirun", "mpiexec"], - "local": [""], - } - - if launcher == "auto": - launcher = detect_launcher() - - def _detect_command(launcher: str) -> str: - if launcher in by_launcher: - if launcher in ["local", "dragon"]: - return "" - - for cmd in by_launcher[launcher]: - if is_valid_cmd(cmd): - return cmd - msg = ( - "Could not automatically detect a run command to use for launcher " - f"{launcher}\nSearched for and could not find the following " - f"commands: {by_launcher[launcher]}" - ) - raise SmartSimError(msg) - - if run_command: - run_command = run_command.lower() - launcher = launcher.lower() - - # detect run_command automatically for all but local launcher - if run_command == "auto": - # no auto detection for local, revert to false - run_command = _detect_command(launcher) - - if launcher == "dragon": - return DragonRunSettings( - exe=exe, exe_args=exe_args, env_vars=env_vars, container=container, **kwargs - ) - - # if user specified and supported or auto detection worked - if run_command and run_command in supported: - return supported[run_command](launcher)( - run_args, env_vars, container=container, **kwargs - ) - - # 1) user specified and not implementation in SmartSim - # 2) user supplied run_command=None - # 3) local launcher being used and default of "auto" was passed. - return RunSettings(run_command, run_args, env_vars, container=container) diff --git a/testing_1/temp_tests/steps_tests.py b/testing_1/temp_tests/steps_tests.py deleted file mode 100644 index b5012170f3..0000000000 --- a/testing_1/temp_tests/steps_tests.py +++ /dev/null @@ -1,110 +0,0 @@ -from smartsim._core.launcher.step import LocalStep, SrunStep, SbatchStep, QsubBatchStep, MpiexecStep, MpirunStep, OrterunStep, BsubBatchStep, JsrunStep, AprunStep -from smartsim.settings import RunSettings, SrunSettings, SbatchSettings, QsubBatchSettings, MpirunSettings, OrterunSettings, BsubBatchSettings, JsrunSettings, AprunSettings -from smartsim.entity import Model -import pytest - -# Test creating a job step -@pytest.mark.parametrize( - "settings_type, step_type", - [ - pytest.param( - AprunSettings, - AprunStep, - id=f"aprun", - ), - pytest.param( - JsrunSettings, - JsrunStep, - id=f"jsrun", - ), - pytest.param( - SrunSettings, - SrunStep, - id="srun", - ), - pytest.param( - RunSettings, - LocalStep, - id="local", - ) - ] -) -def test_instantiate_run_settings( - settings_type, step_type -): - run_settings = settings_type() - run_settings.in_batch = True - model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings) - jobStep = step_type(entity=model, run_settings=model.run_settings) - assert jobStep.run_settings == run_settings - assert jobStep.entity == model - assert jobStep.entity_name == model.name - assert jobStep.cwd == model.path - assert jobStep.step_settings == model.run_settings - -# Test creating a mpi job step -@pytest.mark.parametrize( - "settings_type, step_type", - [ - pytest.param( - OrterunSettings, - OrterunStep, - id="orterun", - ), - pytest.param( - MpirunSettings, - MpirunStep, - id="mpirun", - ), - ] -) -def test_instantiate_mpi_run_settings( - settings_type, step_type -): - run_settings = settings_type(fail_if_missing_exec=False) - run_settings.in_batch = True - model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings) - jobStep = step_type(entity=model, run_settings=model.run_settings) - assert jobStep.run_settings == run_settings - assert jobStep.entity == model - assert jobStep.entity_name == model.name - assert jobStep.cwd == model.path - assert jobStep.step_settings == model.run_settings - -# Test creating a batch job step -@pytest.mark.parametrize( - "settings_type, batch_settings_type, step_type", - [ - pytest.param( - JsrunSettings, - BsubBatchSettings, - BsubBatchStep, - id=f"bsub", - ), - pytest.param( - SrunSettings, - SbatchSettings, - SbatchStep, - id="sbatch", - ), - pytest.param( - RunSettings, - QsubBatchSettings, - QsubBatchStep, - id="qsub", - ) - ] -) -def test_instantiate_batch_settings( - settings_type, batch_settings_type, step_type -): - run_settings = settings_type() - run_settings.in_batch = True - batch_settings = batch_settings_type() - model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings, batch_settings=batch_settings) - jobStep = step_type(entity=model, batch_settings=model.batch_settings) - assert jobStep.batch_settings == batch_settings - assert jobStep.entity == model - assert jobStep.entity_name == model.name - assert jobStep.cwd == model.path - assert jobStep.step_settings == model.batch_settings \ No newline at end of file diff --git a/tests/temp_tests/steps_tests.py b/tests/temp_tests/steps_tests.py index b41f53a50e..b5012170f3 100644 --- a/tests/temp_tests/steps_tests.py +++ b/tests/temp_tests/steps_tests.py @@ -1,30 +1,7 @@ -import pytest - -from smartsim._core.launcher.step import ( - AprunStep, - BsubBatchStep, - JsrunStep, - LocalStep, - MpiexecStep, - MpirunStep, - OrterunStep, - QsubBatchStep, - SbatchStep, - SrunStep, -) +from smartsim._core.launcher.step import LocalStep, SrunStep, SbatchStep, QsubBatchStep, MpiexecStep, MpirunStep, OrterunStep, BsubBatchStep, JsrunStep, AprunStep +from smartsim.settings import RunSettings, SrunSettings, SbatchSettings, QsubBatchSettings, MpirunSettings, OrterunSettings, BsubBatchSettings, JsrunSettings, AprunSettings from smartsim.entity import Model -from smartsim.settings import ( - AprunSettings, - BsubBatchSettings, - JsrunSettings, - MpirunSettings, - OrterunSettings, - QsubBatchSettings, - RunSettings, - SbatchSettings, - SrunSettings, -) - +import pytest # Test creating a job step @pytest.mark.parametrize( @@ -49,15 +26,15 @@ RunSettings, LocalStep, id="local", - ), - ], + ) + ] ) -def test_instantiate_run_settings(settings_type, step_type): +def test_instantiate_run_settings( + settings_type, step_type +): run_settings = settings_type() run_settings.in_batch = True - model = Model( - exe="echo", exe_args="hello", name="model_name", run_settings=run_settings - ) + model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings) jobStep = step_type(entity=model, run_settings=model.run_settings) assert jobStep.run_settings == run_settings assert jobStep.entity == model @@ -65,7 +42,6 @@ def test_instantiate_run_settings(settings_type, step_type): assert jobStep.cwd == model.path assert jobStep.step_settings == model.run_settings - # Test creating a mpi job step @pytest.mark.parametrize( "settings_type, step_type", @@ -80,14 +56,14 @@ def test_instantiate_run_settings(settings_type, step_type): MpirunStep, id="mpirun", ), - ], + ] ) -def test_instantiate_mpi_run_settings(settings_type, step_type): +def test_instantiate_mpi_run_settings( + settings_type, step_type +): run_settings = settings_type(fail_if_missing_exec=False) run_settings.in_batch = True - model = Model( - exe="echo", exe_args="hello", name="model_name", run_settings=run_settings - ) + model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings) jobStep = step_type(entity=model, run_settings=model.run_settings) assert jobStep.run_settings == run_settings assert jobStep.entity == model @@ -95,7 +71,6 @@ def test_instantiate_mpi_run_settings(settings_type, step_type): assert jobStep.cwd == model.path assert jobStep.step_settings == model.run_settings - # Test creating a batch job step @pytest.mark.parametrize( "settings_type, batch_settings_type, step_type", @@ -117,23 +92,19 @@ def test_instantiate_mpi_run_settings(settings_type, step_type): QsubBatchSettings, QsubBatchStep, id="qsub", - ), - ], + ) + ] ) -def test_instantiate_batch_settings(settings_type, batch_settings_type, step_type): +def test_instantiate_batch_settings( + settings_type, batch_settings_type, step_type +): run_settings = settings_type() run_settings.in_batch = True batch_settings = batch_settings_type() - model = Model( - exe="echo", - exe_args="hello", - name="model_name", - run_settings=run_settings, - batch_settings=batch_settings, - ) + model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings, batch_settings=batch_settings) jobStep = step_type(entity=model, batch_settings=model.batch_settings) assert jobStep.batch_settings == batch_settings assert jobStep.entity == model assert jobStep.entity_name == model.name assert jobStep.cwd == model.path - assert jobStep.step_settings == model.batch_settings + assert jobStep.step_settings == model.batch_settings \ No newline at end of file diff --git a/testing_1/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py similarity index 100% rename from testing_1/temp_tests/test_core/test_commands/test_command.py rename to tests/temp_tests/test_core/test_commands/test_command.py diff --git a/testing_1/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py similarity index 100% rename from testing_1/temp_tests/test_core/test_commands/test_commandList.py rename to tests/temp_tests/test_core/test_commands/test_commandList.py diff --git a/testing_1/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_alpsLauncher.py rename to tests/temp_tests/test_settings/test_alpsLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_batchSettings.py rename to tests/temp_tests/test_settings/test_batchSettings.py diff --git a/testing_1/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_dragonLauncher.py rename to tests/temp_tests/test_settings/test_dragonLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_launchSettings.py rename to tests/temp_tests/test_settings/test_launchSettings.py diff --git a/testing_1/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_localLauncher.py rename to tests/temp_tests/test_settings/test_localLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_lsfLauncher.py rename to tests/temp_tests/test_settings/test_lsfLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_lsfScheduler.py rename to tests/temp_tests/test_settings/test_lsfScheduler.py diff --git a/testing_1/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_mpiLauncher.py rename to tests/temp_tests/test_settings/test_mpiLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_palsLauncher.py rename to tests/temp_tests/test_settings/test_palsLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_pbsScheduler.py rename to tests/temp_tests/test_settings/test_pbsScheduler.py diff --git a/testing_1/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_slurmLauncher.py rename to tests/temp_tests/test_settings/test_slurmLauncher.py diff --git a/testing_1/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py similarity index 100% rename from testing_1/temp_tests/test_settings/test_slurmScheduler.py rename to tests/temp_tests/test_settings/test_slurmScheduler.py From fe97911809ff2d6504821e3a55a00a0f9ebdc5af Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 29 May 2024 15:35:42 -0500 Subject: [PATCH 25/43] Matts comments --- conftest.py | 19 ++- smartsim/_core/commands/command.py | 5 +- .../_core/launcher/dragon/dragonLauncher.py | 37 +++-- smartsim/_core/launcher/launcher.py | 5 +- smartsim/_core/launcher/local/local.py | 5 +- smartsim/_core/launcher/lsf/lsfLauncher.py | 27 ++-- smartsim/_core/launcher/pbs/pbsLauncher.py | 30 ++-- .../_core/launcher/slurm/slurmLauncher.py | 27 ++-- smartsim/_core/launcher/step/alpsStep.py | 11 +- smartsim/_core/launcher/step/dragonStep.py | 18 ++- smartsim/_core/launcher/step/localStep.py | 8 +- smartsim/_core/launcher/step/lsfStep.py | 9 +- smartsim/_core/launcher/step/mpiStep.py | 10 +- smartsim/_core/launcher/step/pbsStep.py | 5 +- smartsim/_core/launcher/step/slurmStep.py | 8 +- smartsim/_core/launcher/step/step.py | 6 +- smartsim/database/orchestrator.py | 46 +++++-- smartsim/entity/dbnode.py | 4 +- smartsim/entity/ensemble.py | 6 +- smartsim/entity/entity.py | 4 +- smartsim/entity/model.py | 6 +- smartsim/experiment.py | 123 ++--------------- smartsim/settings/batchCommand.py | 6 +- smartsim/settings/batchSettings.py | 76 +++++----- smartsim/settings/common.py | 2 +- smartsim/settings/launchCommand.py | 4 +- smartsim/settings/launchSettings.py | 130 +++++++++--------- smartsim/settings/translators/batch/lsf.py | 4 +- smartsim/settings/translators/batch/pbs.py | 6 +- smartsim/settings/translators/batch/slurm.py | 2 +- .../settings/translators/launch/__init__.py | 22 +-- smartsim/settings/translators/launch/alps.py | 2 +- .../settings/translators/launch/dragon.py | 2 +- smartsim/settings/translators/launch/local.py | 2 +- smartsim/settings/translators/launch/lsf.py | 7 +- smartsim/settings/translators/launch/mpi.py | 2 +- smartsim/settings/translators/launch/pals.py | 2 +- smartsim/settings/translators/launch/slurm.py | 2 +- .../translators/launchArgTranslator.py | 2 +- smartsim/wlm/slurm.py | 4 +- .../test_core/test_commands/test_command.py | 2 +- .../test_commands/test_commandList.py | 2 +- .../test_settings/test_alpsLauncher.py | 17 +-- .../test_settings/test_batchSettings.py | 26 +++- .../test_settings/test_dragonLauncher.py | 11 +- .../test_settings/test_launchSettings.py | 26 +++- .../test_settings/test_localLauncher.py | 11 +- .../test_settings/test_lsfLauncher.py | 16 +-- .../test_settings/test_lsfScheduler.py | 18 +-- .../test_settings/test_mpiLauncher.py | 14 +- .../test_settings/test_palsLauncher.py | 11 +- .../test_settings/test_pbsScheduler.py | 27 ++-- .../test_settings/test_slurmLauncher.py | 14 +- .../test_settings/test_slurmScheduler.py | 24 ++-- 54 files changed, 462 insertions(+), 453 deletions(-) diff --git a/conftest.py b/conftest.py index b0457522c0..64dad911d3 100644 --- a/conftest.py +++ b/conftest.py @@ -59,16 +59,15 @@ from smartsim.entity import Model from smartsim.error import SSConfigError, SSInternalError from smartsim.log import get_logger -from smartsim.settings import ( - AprunSettings, - DragonRunSettings, - JsrunSettings, - MpiexecSettings, - MpirunSettings, - PalsMpiexecSettings, - RunSettings, - SrunSettings, -) +# Mock imports +class AprunSettings: pass +class DragonRunSettings: pass +class JsrunSettings: pass +class MpiexecSettings: pass +class MpirunSettings: pass +class PalsMpiexecSettings: pass +class RunSettings: pass +class SrunSettings: pass logger = get_logger(__name__) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index cdde395acb..83b2bcb005 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from collections.abc import MutableSequence -from ...settingshold.launchCommand import LauncherType +from ...settings.launchCommand import LauncherType import typing as t class Command(MutableSequence): @@ -77,7 +77,6 @@ def insert(self, idx: int, value: str) -> None: self._command.insert(idx, value) def __str__(self) -> str: # pragma: no cover - string = "" - string += f"\nLauncher: {self.launcher.value}\n" + string = f"\nLauncher: {self.launcher.value}\n" string += f"Command: {' '.join(str(cmd) for cmd in self.command)}" return string \ No newline at end of file diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 17b47e3090..f4f89974c2 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -32,13 +32,20 @@ from ...._core.launcher.stepMapping import StepMap from ....error import LauncherError, SmartSimError from ....log import get_logger -from ....settings import ( - DragonRunSettings, - QsubBatchSettings, - RunSettings, - SbatchSettings, - SettingsBase, -) +# from ....settings import ( +# DragonRunSettings, +# QsubBatchSettings, +# RunSettings, +# SbatchSettings, +# SettingsBase, +# ) +# Mock imports for now +class DragonRunSettings: pass +class QsubBatchSettings: pass +class RunSettings: pass +class SbatchSettings: pass +class SettingsBase: pass + from ....status import SmartSimStatus from ...schemas import ( DragonRunRequest, @@ -89,15 +96,17 @@ def cleanup(self) -> None: self._connector.cleanup() # RunSettings types supported by this launcher + # t.Type[SettingsBase] @property - def supported_rs(self) -> t.Dict[t.Type[SettingsBase], t.Type[Step]]: + def supported_rs(self) -> t.Dict[t.Type[Step]]: # RunSettings types supported by this launcher - return { - DragonRunSettings: DragonStep, - SbatchSettings: DragonBatchStep, - QsubBatchSettings: DragonBatchStep, - RunSettings: LocalStep, - } + pass + # return { + # DragonRunSettings: DragonStep, + # SbatchSettings: DragonBatchStep, + # QsubBatchSettings: DragonBatchStep, + # RunSettings: LocalStep, + # } def add_step_to_mapping_table(self, name: str, step_map: StepMap) -> None: super().add_step_to_mapping_table(name, step_map) diff --git a/smartsim/_core/launcher/launcher.py b/smartsim/_core/launcher/launcher.py index bbc9b59d62..a8c3d12ed4 100644 --- a/smartsim/_core/launcher/launcher.py +++ b/smartsim/_core/launcher/launcher.py @@ -30,7 +30,10 @@ from ..._core.launcher.stepMapping import StepMap from ...entity import SmartSimEntity from ...error import AllocationError, LauncherError, SSUnsupportedError -from ...settings import SettingsBase +# from ...settings import SettingsBase +# Mock imports +class SettingsBase: pass + from .step import Step from .stepInfo import StepInfo, UnmanagedStepInfo from .stepMapping import StepMapping diff --git a/smartsim/_core/launcher/local/local.py b/smartsim/_core/launcher/local/local.py index 2b3f479971..fd47737ad8 100644 --- a/smartsim/_core/launcher/local/local.py +++ b/smartsim/_core/launcher/local/local.py @@ -27,7 +27,10 @@ import typing as t from ....entity import SmartSimEntity -from ....settings import RunSettings, SettingsBase +# from ....settings import RunSettings, SettingsBase +# Mock imports +class RunSettings: pass +class SettingsBase: pass from ..launcher import Launcher from ..step import LocalStep, Step from ..stepInfo import StepInfo, UnmanagedStepInfo diff --git a/smartsim/_core/launcher/lsf/lsfLauncher.py b/smartsim/_core/launcher/lsf/lsfLauncher.py index e0ad808ed8..027b5e4e66 100644 --- a/smartsim/_core/launcher/lsf/lsfLauncher.py +++ b/smartsim/_core/launcher/lsf/lsfLauncher.py @@ -29,15 +29,24 @@ from ....error import LauncherError from ....log import get_logger -from ....settings import ( - BsubBatchSettings, - JsrunSettings, - MpiexecSettings, - MpirunSettings, - OrterunSettings, - RunSettings, - SettingsBase, -) +# from ....settings import ( +# BsubBatchSettings, +# JsrunSettings, +# MpiexecSettings, +# MpirunSettings, +# OrterunSettings, +# RunSettings, +# SettingsBase, +# ) +# Mock imports +class BsubBatchSettings: pass +class JsrunSettings: pass +class MpiexecSettings: pass +class MpirunSettings: pass +class OrterunSettings: pass +class RunSettings: pass +class SettingsBase: pass + from ....status import SmartSimStatus from ...config import CONFIG from ..launcher import WLMLauncher diff --git a/smartsim/_core/launcher/pbs/pbsLauncher.py b/smartsim/_core/launcher/pbs/pbsLauncher.py index 8c2099a8bc..d870d40473 100644 --- a/smartsim/_core/launcher/pbs/pbsLauncher.py +++ b/smartsim/_core/launcher/pbs/pbsLauncher.py @@ -29,16 +29,26 @@ from ....error import LauncherError from ....log import get_logger -from ....settings import ( - AprunSettings, - MpiexecSettings, - MpirunSettings, - OrterunSettings, - PalsMpiexecSettings, - QsubBatchSettings, - RunSettings, - SettingsBase, -) +# from ....settings import ( +# AprunSettings, +# MpiexecSettings, +# MpirunSettings, +# OrterunSettings, +# PalsMpiexecSettings, +# QsubBatchSettings, +# RunSettings, +# SettingsBase, +# ) +# Mock imports +class AprunSettings: pass +class MpiexecSettings: pass +class MpirunSettings: pass +class OrterunSettings: pass +class PalsMpiexecSettings: pass +class QsubBatchSettings: pass +class RunSettings: pass +class SettingsBase: pass + from ....status import SmartSimStatus from ...config import CONFIG from ..launcher import WLMLauncher diff --git a/smartsim/_core/launcher/slurm/slurmLauncher.py b/smartsim/_core/launcher/slurm/slurmLauncher.py index 2e41023919..fca108ff01 100644 --- a/smartsim/_core/launcher/slurm/slurmLauncher.py +++ b/smartsim/_core/launcher/slurm/slurmLauncher.py @@ -31,15 +31,24 @@ from ....error import LauncherError from ....log import get_logger -from ....settings import ( - MpiexecSettings, - MpirunSettings, - OrterunSettings, - RunSettings, - SbatchSettings, - SettingsBase, - SrunSettings, -) +# from ....settings import ( +# MpiexecSettings, +# MpirunSettings, +# OrterunSettings, +# RunSettings, +# SbatchSettings, +# SettingsBase, +# SrunSettings, +# ) +# Mock imports +class SrunSettings: pass +class MpiexecSettings: pass +class MpirunSettings: pass +class OrterunSettings: pass +class SbatchSettings: pass +class RunSettings: pass +class SettingsBase: pass + from ....status import SmartSimStatus from ...config import CONFIG from ..launcher import WLMLauncher diff --git a/smartsim/_core/launcher/step/alpsStep.py b/smartsim/_core/launcher/step/alpsStep.py index 9b744498bd..27359852d1 100644 --- a/smartsim/_core/launcher/step/alpsStep.py +++ b/smartsim/_core/launcher/step/alpsStep.py @@ -32,15 +32,18 @@ from ....entity import DBNode, Model from ....error import AllocationError from ....log import get_logger -from ....settings import AprunSettings, RunSettings, Singularity +# from ....settings import AprunSettings, RunSettings, Singularity +# Mock imports for now +class AprunSettings: pass +class RunSettings: pass +class Singularity: pass from .step import Step, proxyable_launch_cmd logger = get_logger(__name__) - class AprunStep(Step): def __init__( - self, entity: t.Union[Model, DBNode], run_settings: AprunSettings + self, entity: t.Union[Model, DBNode], run_settings: str ) -> None: """Initialize a ALPS aprun job step @@ -89,7 +92,7 @@ def get_launch_cmd(self) -> t.List[str]: launch_script_path = self.get_colocated_launch_script() aprun_cmd.extend([bash, launch_script_path]) - if isinstance(self.run_settings.container, Singularity): + if isinstance(self.run_settings.container, str): #changed from Singularity to str # pylint: disable-next=protected-access aprun_cmd += self.run_settings.container._container_cmds(self.cwd) diff --git a/smartsim/_core/launcher/step/dragonStep.py b/smartsim/_core/launcher/step/dragonStep.py index 036a9e5654..75ab0fb78c 100644 --- a/smartsim/_core/launcher/step/dragonStep.py +++ b/smartsim/_core/launcher/step/dragonStep.py @@ -33,12 +33,18 @@ from ...._core.schemas.dragonRequests import DragonRunRequest, request_registry from ....error.errors import SSUnsupportedError from ....log import get_logger -from ....settings import ( - DragonRunSettings, - QsubBatchSettings, - SbatchSettings, - Singularity, -) +# from ....settings import ( +# DragonRunSettings, +# QsubBatchSettings, +# SbatchSettings, +# Singularity, +# ) +# Temp mock imports +class DragonRunSettings: pass +class QsubBatchSettings: pass +class SbatchSettings: pass +class Singularity: pass + from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/localStep.py b/smartsim/_core/launcher/step/localStep.py index 57e2a87837..679d61a55d 100644 --- a/smartsim/_core/launcher/step/localStep.py +++ b/smartsim/_core/launcher/step/localStep.py @@ -29,10 +29,14 @@ import typing as t from ....entity import DBNode, Model -from ....settings import Singularity -from ....settings.base import RunSettings +# from ....settings import Singularity +# from ....settings.base import RunSettings from .step import Step, proxyable_launch_cmd +# Temperary mock imports +class Singularity: pass +class RunSettings: pass + class LocalStep(Step): def __init__(self, entity: t.Union[Model, DBNode], run_settings: RunSettings): diff --git a/smartsim/_core/launcher/step/lsfStep.py b/smartsim/_core/launcher/step/lsfStep.py index 6782fbd273..7a07c852e9 100644 --- a/smartsim/_core/launcher/step/lsfStep.py +++ b/smartsim/_core/launcher/step/lsfStep.py @@ -31,8 +31,13 @@ from ....entity import DBNode, Model from ....error import AllocationError from ....log import get_logger -from ....settings import BsubBatchSettings, JsrunSettings -from ....settings.base import RunSettings +# from ....settings import BsubBatchSettings, JsrunSettings +# from ....settings.base import RunSettings +# Temp mock imports +class BsubBatchSettings: pass +class JsrunSettings: pass +class RunSettings: pass + from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/mpiStep.py b/smartsim/_core/launcher/step/mpiStep.py index 1cbab9fd8f..746f7e48c0 100644 --- a/smartsim/_core/launcher/step/mpiStep.py +++ b/smartsim/_core/launcher/step/mpiStep.py @@ -32,8 +32,14 @@ from ....entity import DBNode, Model from ....error import AllocationError, SmartSimError from ....log import get_logger -from ....settings import MpiexecSettings, MpirunSettings, OrterunSettings -from ....settings.base import RunSettings +# from ....settings import MpiexecSettings, MpirunSettings, OrterunSettings +# from ....settings.base import RunSettings +# Temp mock imports +class MpiexecSettings: pass +class MpirunSettings: pass +class OrterunSettings: pass +class RunSettings: pass + from .step import Step, proxyable_launch_cmd logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/pbsStep.py b/smartsim/_core/launcher/step/pbsStep.py index 9177dd6d56..81f5f73336 100644 --- a/smartsim/_core/launcher/step/pbsStep.py +++ b/smartsim/_core/launcher/step/pbsStep.py @@ -28,7 +28,10 @@ from ....entity import DBNode, Model from ....log import get_logger -from ....settings import QsubBatchSettings +# from ....settings import QsubBatchSettings +# Temp mock imports +class QsubBatchSettings: pass + from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/slurmStep.py b/smartsim/_core/launcher/step/slurmStep.py index 5711a56942..3b47cc7d94 100644 --- a/smartsim/_core/launcher/step/slurmStep.py +++ b/smartsim/_core/launcher/step/slurmStep.py @@ -32,7 +32,13 @@ from ....entity import DBNode, Ensemble, Model from ....error import AllocationError from ....log import get_logger -from ....settings import RunSettings, SbatchSettings, Singularity, SrunSettings +# from ....settings import RunSettings, SbatchSettings, Singularity, SrunSettings +# Temp mock imports +class RunSettings: pass +class SbatchSettings: pass +class Singularity: pass +class SrunSettings: pass + from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/step.py b/smartsim/_core/launcher/step/step.py index 46c7478db0..9cdd3721b4 100644 --- a/smartsim/_core/launcher/step/step.py +++ b/smartsim/_core/launcher/step/step.py @@ -39,7 +39,11 @@ from ....entity import DBNode, Ensemble, Model from ....log import get_logger -from ....settings.base import RunSettings, SettingsBase +# from ....settings.base import RunSettings, SettingsBase +# Temp mock imports +class RunSettings: pass +class SettingsBase: pass + from ...utils.helpers import encode_cmd, get_base_36_repr from ..colocated import write_colocated_launch_script diff --git a/smartsim/database/orchestrator.py b/smartsim/database/orchestrator.py index 3309c591cc..6a40fe49b5 100644 --- a/smartsim/database/orchestrator.py +++ b/smartsim/database/orchestrator.py @@ -50,20 +50,38 @@ ) from ..log import get_logger from ..servertype import CLUSTERED, STANDALONE -from ..settings import ( - AprunSettings, - BsubBatchSettings, - JsrunSettings, - MpiexecSettings, - MpirunSettings, - OrterunSettings, - PalsMpiexecSettings, - QsubBatchSettings, - SbatchSettings, - SrunSettings, -) -from ..settings.base import BatchSettings, RunSettings -from ..settings.settings import create_batch_settings, create_run_settings +# from ..settings import ( +# AprunSettings, +# BsubBatchSettings, +# JsrunSettings, +# MpiexecSettings, +# MpirunSettings, +# OrterunSettings, +# PalsMpiexecSettings, +# QsubBatchSettings, +# SbatchSettings, +# SrunSettings, +# ) +# Mock imports +class AprunSettings: pass +class BsubBatchSettings: pass +class JsrunSettings: pass +class MpiexecSettings: pass +class OrterunSettings: pass +class PalsMpiexecSettings: pass +class QsubBatchSettings: pass +class SbatchSettings: pass +class SrunSettings: pass + +# from ..settings.base import BatchSettings, RunSettings +# Mock imports +class BatchSettings: pass +class RunSettings: pass + +# from ..settings.settings import create_batch_settings, create_run_settings +# Mock functions +def create_batch_settings() -> None: ... +def create_run_settings() -> None: ... from ..wlm import detect_launcher logger = get_logger(__name__) diff --git a/smartsim/entity/dbnode.py b/smartsim/entity/dbnode.py index 9e370f64d9..afeed68dfc 100644 --- a/smartsim/entity/dbnode.py +++ b/smartsim/entity/dbnode.py @@ -37,7 +37,7 @@ from .._core.utils.helpers import expand_exe_path from ..error import SSDBFilesNotParseable from ..log import get_logger -from ..settings.base import RunSettings +# from ..settings.base import RunSettings from .entity import SmartSimEntity logger = get_logger(__name__) @@ -58,7 +58,7 @@ def __init__( path: str, exe: str, exe_args: t.List[str], - run_settings: RunSettings, + # run_settings: RunSettings, ports: t.List[int], output_files: t.List[str], db_identifier: str = "", diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index e3ddf38cf4..050a545620 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -40,7 +40,7 @@ UserStrategyError, ) from ..log import get_logger -from ..settings.base import BatchSettings, RunSettings +# from ..settings.base import BatchSettings, RunSettings from .dbobject import DBModel, DBScript from .entity import SmartSimEntity from .entityList import EntityList @@ -67,8 +67,8 @@ def __init__( exe_args: t.Optional[t.List[str]] = None, path: t.Optional[str] = getcwd(), params_as_args: t.Optional[t.List[str]] = None, - batch_settings: t.Optional[BatchSettings] = None, - run_settings: t.Optional[RunSettings] = None, + # batch_settings: t.Optional[BatchSettings] = None, + # run_settings: t.Optional[RunSettings] = None, perm_strat: str = "all_perm", **kwargs: t.Any, ) -> None: diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index b68ea017fd..6d08efbf60 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -26,9 +26,9 @@ import typing as t -if t.TYPE_CHECKING: +# if t.TYPE_CHECKING: # pylint: disable-next=unused-import - import smartsim.settings.base + # import smartsim.settings.base class TelemetryConfiguration: diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 96ab4a780c..ecf96c8cb8 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -39,7 +39,7 @@ from .._core.utils.helpers import cat_arg_and_value, expand_exe_path from ..error import EntityExistsError, SSUnsupportedError from ..log import get_logger -from ..settings.base import BatchSettings, RunSettings +# from ..settings.base import BatchSettings, RunSettings from .dbobject import DBModel, DBScript from .entity import SmartSimEntity from .files import EntityFiles @@ -52,12 +52,12 @@ def __init__( self, name: str, exe: str, - run_settings: RunSettings, + # run_settings: RunSettings, params: t.Optional[t.Dict[str, str]] = None, exe_args: t.Optional[t.List[str]] = None, path: t.Optional[str] = getcwd(), params_as_args: t.Optional[t.List[str]] = None, - batch_settings: t.Optional[BatchSettings] = None, + # batch_settings: t.Optional[BatchSettings] = None, ): """Initialize a ``Model`` diff --git a/smartsim/experiment.py b/smartsim/experiment.py index d8163cf711..dc00e581b1 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -48,7 +48,11 @@ ) from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer -from .settings import Container, base, settings +# from .settings import Container, base, settings +# Mock imports +class Container: pass +class BatchSettings: pass +class RunSettings: pass from .wlm import detect_launcher logger = get_logger(__name__) @@ -434,8 +438,8 @@ def create_ensemble( exe: t.Optional[str] = None, exe_args: t.Optional[t.List[str]] = None, params: t.Optional[t.Dict[str, t.Any]] = None, - batch_settings: t.Optional[base.BatchSettings] = None, - run_settings: t.Optional[base.RunSettings] = None, + batch_settings: t.Optional[BatchSettings] = None, + run_settings: t.Optional[RunSettings] = None, replicas: t.Optional[int] = None, perm_strategy: str = "all_perm", path: t.Optional[str] = None, @@ -513,12 +517,12 @@ def create_model( self, name: str, exe: str, - run_settings: base.RunSettings, + run_settings: RunSettings, exe_args: t.Optional[t.List[str]] = None, params: t.Optional[t.Dict[str, t.Any]] = None, path: t.Optional[str] = None, enable_key_prefixing: bool = False, - batch_settings: t.Optional[base.BatchSettings] = None, + batch_settings: t.Optional[BatchSettings] = None, ) -> Model: """Create a general purpose ``Model`` @@ -623,115 +627,6 @@ def create_model( logger.error(e) raise - @_contextualize - def create_run_settings( - self, - run_command: str = "auto", - run_args: t.Optional[t.Dict[str, t.Union[int, str, float, None]]] = None, - env_vars: t.Optional[t.Dict[str, t.Optional[str]]] = None, - container: t.Optional[Container] = None, - **kwargs: t.Any, - ) -> settings.RunSettings: - """Create a ``RunSettings`` instance. - - run_command="auto" will attempt to automatically - match a run command on the system with a ``RunSettings`` - class in SmartSim. If found, the class corresponding - to that run_command will be created and returned. - - If the local launcher is being used, auto detection will - be turned off. - - If a recognized run command is passed, the ``RunSettings`` - instance will be a child class such as ``SrunSettings`` - - If not supported by smartsim, the base ``RunSettings`` class - will be created and returned with the specified run_command and run_args - will be evaluated literally. - - Run Commands with implemented helper classes: - - aprun (ALPS) - - srun (SLURM) - - mpirun (OpenMPI) - - jsrun (LSF) - - :param run_command: command to run the executable - :param exe: executable to run - :param exe_args: arguments to pass to the executable - :param run_args: arguments to pass to the ``run_command`` - :param env_vars: environment variables to pass to the executable - :param container: if execution environment is containerized - :return: the created ``RunSettings`` - """ - - try: - return settings.create_run_settings( - self._launcher, - run_command=run_command, - run_args=run_args, - env_vars=env_vars, - container=container, - **kwargs, - ) - except SmartSimError as e: - logger.error(e) - raise - - @_contextualize - def create_batch_settings( - self, - nodes: int = 1, - time: str = "", - queue: str = "", - account: str = "", - batch_args: t.Optional[t.Dict[str, str]] = None, - **kwargs: t.Any, - ) -> base.BatchSettings: - """Create a ``BatchSettings`` instance - - Batch settings parameterize batch workloads. The result of this - function can be passed to the ``Ensemble`` initialization. - - the `batch_args` parameter can be used to pass in a dictionary - of additional batch command arguments that aren't supported through - the smartsim interface - - - .. highlight:: python - .. code-block:: python - - # i.e. for Slurm - batch_args = { - "distribution": "block" - "exclusive": None - } - bs = exp.create_batch_settings(nodes=3, - time="10:00:00", - batch_args=batch_args) - bs.set_account("default") - - :param nodes: number of nodes for batch job - :param time: length of batch job - :param queue: queue or partition (if slurm) - :param account: user account name for batch system - :param batch_args: additional batch arguments - :return: a newly created BatchSettings instance - :raises SmartSimError: if batch creation fails - """ - try: - return settings.create_batch_settings( - self._launcher, - nodes=nodes, - time=time, - queue=queue, - account=account, - batch_args=batch_args, - **kwargs, - ) - except SmartSimError as e: - logger.error(e) - raise - @_contextualize def create_database( self, diff --git a/smartsim/settings/batchCommand.py b/smartsim/settings/batchCommand.py index b92d668111..8275c1728e 100644 --- a/smartsim/settings/batchCommand.py +++ b/smartsim/settings/batchCommand.py @@ -30,6 +30,6 @@ class SchedulerType(Enum): """ Schedulers that are supported by SmartSim. """ - SlurmScheduler = "sbatch" - PbsScheduler = "qsub" - LsfScheduler = "bsub" \ No newline at end of file + SlurmScheduler = "slurm" + PbsScheduler = "pbs" + LsfScheduler = "lsf" \ No newline at end of file diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 4b0288198e..285a983443 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -43,20 +43,15 @@ class BatchSettings(BaseSettings): def __init__( self, - batch_scheduler: t.Union[SchedulerType, BatchArgTranslator], + batch_scheduler: t.Union[SchedulerType, str], scheduler_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, env_vars: t.Optional[StringArgument] = None, ) -> None: - if isinstance(batch_scheduler, BatchArgTranslator): - self.arg_translator = batch_scheduler - elif batch_scheduler.value == 'sbatch': - self.arg_translator = SlurmBatchArgTranslator() - elif batch_scheduler.value == 'bsub': - self.arg_translator = BsubBatchArgTranslator() - elif batch_scheduler.value == 'qsub': - self.arg_translator = QsubBatchArgTranslator() - else: - raise ValueError(f"'{batch_scheduler}' is not valid input.") + try: + self._batch_scheduler = SchedulerType(batch_scheduler) + except KeyError: + raise ValueError(f"Invalid scheduler type: {batch_scheduler}") + self._arg_translator = self._get_scheduler() if env_vars: validate_env_vars(env_vars) @@ -66,6 +61,14 @@ def __init__( validate_args(scheduler_args) self.scheduler_args = scheduler_args or {} + @property + def batch_scheduler(self): + return self._batch_scheduler + + @property + def arg_translator(self): + return self._arg_translator + @property def scheduler_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: """Retrieve attached batch arguments @@ -82,6 +85,16 @@ def scheduler_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> N """ self._scheduler_args = copy.deepcopy(value) if value else {} + def _get_scheduler(self) -> BatchArgTranslator: + """ Map the Scheduler to the BatchArgTranslator + """ + if self._batch_scheduler.value == 'slurm': + return SlurmBatchArgTranslator() + elif self._batch_scheduler.value == 'lsf': + return BsubBatchArgTranslator() + elif self._batch_scheduler.value == 'pbs': + return QsubBatchArgTranslator() + def scheduler_str(self) -> str: """ Get the string representation of the scheduler """ @@ -97,8 +110,7 @@ def set_walltime(self, walltime: str) -> None: # TODO check for formatting here args = self.arg_translator.set_walltime(walltime) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job @@ -107,8 +119,7 @@ def set_nodes(self, num_nodes: int) -> None: """ args = self.arg_translator.set_nodes(num_nodes) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_account(self, account: str) -> None: """Set the account for this batch job @@ -117,8 +128,7 @@ def set_account(self, account: str) -> None: """ args = self.arg_translator.set_account(account) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_partition(self, partition: str) -> None: """Set the partition for the batch job @@ -127,8 +137,7 @@ def set_partition(self, partition: str) -> None: """ args = self.arg_translator.set_partition(partition) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_queue(self, queue: str) -> None: """alias for set_partition @@ -139,8 +148,7 @@ def set_queue(self, queue: str) -> None: """ args = self.arg_translator.set_queue(queue) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_cpus_per_task(self, cpus_per_task: int) -> None: """Set the number of cpus to use per task @@ -151,8 +159,7 @@ def set_cpus_per_task(self, cpus_per_task: int) -> None: """ args = self.arg_translator.set_cpus_per_task(cpus_per_task) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job @@ -162,8 +169,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """ args = self.arg_translator.set_hostlist(host_list) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_smts(self, smts: int) -> None: """Set SMTs @@ -176,8 +182,7 @@ def set_smts(self, smts: int) -> None: """ args = self.arg_translator.set_smts(smts) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_project(self, project: str) -> None: """Set the project @@ -188,8 +193,7 @@ def set_project(self, project: str) -> None: """ args = self.arg_translator.set_project(project) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_tasks(self, tasks: int) -> None: """Set the number of tasks for this job @@ -200,8 +204,7 @@ def set_tasks(self, tasks: int) -> None: """ args = self.arg_translator.set_tasks(tasks) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def set_ncpus(self, num_cpus: int) -> None: """Set the number of cpus obtained in each node. @@ -214,23 +217,24 @@ def set_ncpus(self, num_cpus: int) -> None: """ args = self.arg_translator.set_ncpus(num_cpus) if args: - for key, value in args.items(): - self.set(key, value) + self.update_scheduler_args(args) def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview """ return self.arg_translator.format_batch_args(self.scheduler_args) + def update_scheduler_args(self, args: t.Mapping[str, int | str | float | None]) -> None: + self.scheduler_args.update(args) + def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: # Store custom arguments in the launcher_args self.scheduler_args[key] = arg def __str__(self) -> str: # pragma: no-cover - string = "" - string += f"\Scheduler: {self.arg_translator.scheduler_str}" + string = f"\nScheduler: {self.arg_translator.scheduler_str}" if self.scheduler_args: - string += f"\Scheduler Arguments:\n{fmt_dict(self.scheduler_args)}" + string += f"\nScheduler Arguments:\n{fmt_dict(self.scheduler_args)}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" return string \ No newline at end of file diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index d3daf629c2..e2fa5d658a 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -36,7 +36,7 @@ def validate_env_vars(env_vars: StringArgument) -> None: if not isinstance(value, str) and value is not None: raise TypeError(f"Value for '{key}' must be a string.") -def validate_args(launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> None: +def validate_args(launch_args: t.Mapping[str, t.Union[str,int,float,None]]) -> None: """Validate that user passed launch args and scheduler args are of correct type. """ for key, value in launch_args.items(): diff --git a/smartsim/settings/launchCommand.py b/smartsim/settings/launchCommand.py index fa004677a3..b442bfad37 100644 --- a/smartsim/settings/launchCommand.py +++ b/smartsim/settings/launchCommand.py @@ -33,9 +33,9 @@ class LauncherType(Enum): DragonLauncher = "dragon" SlurmLauncher = "slurm" PalsLauncher = "pals" - AlpsLauncher = "aprun" + AlpsLauncher = "alps" LocalLauncher = "local" MpiexecLauncher = "mpiexec" MpirunLauncher = "mpirun" OrterunLauncher = "orterun" - LsfLauncher = "jsrun" \ No newline at end of file + LsfLauncher = "lsf" \ No newline at end of file diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index c059c2425d..15ea8e9a22 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -48,32 +48,15 @@ class LaunchSettings(BaseSettings): def __init__( self, - launcher: t.Union[LauncherType, LaunchArgTranslator], + launcher: t.Union[LauncherType, str], launcher_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, env_vars: t.Optional[StringArgument] = None, ) -> None: - if isinstance(launcher, LaunchArgTranslator): - self.arg_translator = launcher - elif launcher.value == 'slurm': - self.arg_translator = SlurmArgTranslator() - elif launcher.value == 'mpiexec': - self.arg_translator = MpiexecArgTranslator() - elif launcher.value == 'mpirun': - self.arg_translator = MpiArgTranslator() - elif launcher.value == 'orterun': - self.arg_translator = OrteArgTranslator() - elif launcher.value == 'aprun': - self.arg_translator = AprunArgTranslator() - elif launcher.value == 'jsrun': - self.arg_translator = JsrunArgTranslator() - elif launcher.value == 'pals': - self.arg_translator = PalsMpiexecArgTranslator() - elif launcher.value == 'dragon': - self.arg_translator = DragonArgTranslator() - elif launcher.value == 'local': - self.arg_translator = LocalArgTranslator() - else: - raise ValueError(f"'{launcher}' is not valid input.") + try: + self._launcher = LauncherType(launcher) + except KeyError: + raise ValueError(f"Invalid launcher type: {launcher}") + self._arg_translator = self._get_launcher() if env_vars: validate_env_vars(env_vars) @@ -82,9 +65,14 @@ def __init__( if launcher_args: validate_args(launcher_args) self.launcher_args = launcher_args or {} - - # Set the reserved launch arguments per LauncherType - self._reserved_launch_args = self.arg_translator._set_reserved_launch_args() + + @property + def launcher(self): + return self._launcher + + @property + def arg_translator(self): + return self._arg_translator @property def launcher_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: @@ -117,6 +105,32 @@ def env_vars(self, value: StringArgument) -> None: :param value: environment variables """ self._env_vars = copy.deepcopy(value) + + @property + def reserved_launch_args(self): + return self.arg_translator.set_reserved_launch_args() + + def _get_launcher(self) -> LaunchArgTranslator: + """ Map the Launcher to the LaunchArgTranslator + """ + if self._launcher.value == 'slurm': + return SlurmArgTranslator() + elif self._launcher.value == 'mpiexec': + return MpiexecArgTranslator() + elif self._launcher.value == 'mpirun': + return MpiArgTranslator() + elif self._launcher.value == 'orterun': + return OrteArgTranslator() + elif self._launcher.value == 'alps': + return AprunArgTranslator() + elif self._launcher.value == 'lsf': + return JsrunArgTranslator() + elif self._launcher.value == 'pals': + return PalsMpiexecArgTranslator() + elif self._launcher.value == 'dragon': + return DragonArgTranslator() + elif self._launcher.value == 'local': + return LocalArgTranslator() def launcher_str(self) -> str: """ Get the string representation of the launcher @@ -132,8 +146,7 @@ def set_nodes(self, nodes: int) -> None: args = self.arg_translator.set_nodes(nodes) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """ Specify the hostlist for this job @@ -144,8 +157,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: args = self.arg_translator.set_hostlist(host_list) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_hostlist_from_file(self, file_path: str) -> None: """ Use the contents of a file to set the node list @@ -156,8 +168,7 @@ def set_hostlist_from_file(self, file_path: str) -> None: args = self.arg_translator.set_hostlist_from_file(file_path) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: """ Specify a list of hosts to exclude for launching this job @@ -168,8 +179,7 @@ def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: args = self.arg_translator.set_excluded_hosts(host_list) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_cpus_per_task(self, cpus_per_task: int) -> None: """ Set the number of cpus to use per task @@ -180,8 +190,7 @@ def set_cpus_per_task(self, cpus_per_task: int) -> None: args = self.arg_translator.set_cpus_per_task(cpus_per_task) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_tasks(self, tasks: int) -> None: """ Set the number of tasks for this job @@ -190,8 +199,7 @@ def set_tasks(self, tasks: int) -> None: """ args = self.arg_translator.set_tasks(tasks) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_tasks_per_node(self, tasks_per_node: int) -> None: """ Set the number of tasks per node for this job @@ -200,8 +208,7 @@ def set_tasks_per_node(self, tasks_per_node: int) -> None: """ args = self.arg_translator.set_tasks_per_node(tasks_per_node) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: """ Bind by setting CPU masks on tasks @@ -210,8 +217,7 @@ def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: """ args = self.arg_translator.set_cpu_bindings(bindings) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_memory_per_node(self, memory_per_node: int) -> None: """ Specify the real memory required per node @@ -220,8 +226,7 @@ def set_memory_per_node(self, memory_per_node: int) -> None: """ args = self.arg_translator.set_memory_per_node(memory_per_node) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_executable_broadcast(self, dest_path: str) -> None: """ Copy executable file to allocated compute nodes @@ -230,8 +235,7 @@ def set_executable_broadcast(self, dest_path: str) -> None: """ args = self.arg_translator.set_executable_broadcast(dest_path) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: """Specify the node feature for this job @@ -241,8 +245,7 @@ def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: """ args = self.arg_translator.set_node_feature(feature_list) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_walltime(self, walltime: str) -> None: """Set the walltime of the job @@ -251,8 +254,7 @@ def set_walltime(self, walltime: str) -> None: """ args = self.arg_translator.set_walltime(walltime) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_binding(self, binding: str) -> None: """Set binding @@ -263,8 +265,7 @@ def set_binding(self, binding: str) -> None: """ args = self.arg_translator.set_binding(binding) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_cpu_binding_type(self, bind_type: str) -> None: """Specifies the cores to which MPI processes are bound @@ -275,8 +276,7 @@ def set_cpu_binding_type(self, bind_type: str) -> None: """ args = self.arg_translator.set_cpu_binding_type(bind_type) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_task_map(self, task_mapping: str) -> None: """Set ``mpirun`` task mapping @@ -289,8 +289,7 @@ def set_task_map(self, task_mapping: str) -> None: """ args = self.arg_translator.set_task_map(task_mapping) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_het_group(self, het_group: t.Iterable[int]) -> None: """Set the heterogeneous group for this job @@ -301,8 +300,7 @@ def set_het_group(self, het_group: t.Iterable[int]) -> None: """ args = self.arg_translator.set_het_group(het_group) if args: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) def set_verbose_launch(self, verbose: bool) -> None: """Set the job to run in verbose mode @@ -313,8 +311,7 @@ def set_verbose_launch(self, verbose: bool) -> None: """ args = self.arg_translator.set_verbose_launch(verbose) if args and verbose: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) if args and not verbose: self.launcher_args.pop(next(iter(args))) @@ -327,8 +324,7 @@ def set_quiet_launch(self, quiet: bool) -> None: """ args = self.arg_translator.set_quiet_launch(quiet) if args and quiet: - for key, value in args.items(): - self.set(key, value) + self.update_launcher_args(args) if args and not quiet: self.launcher_args.pop(next(iter(args))) @@ -382,12 +378,15 @@ def update_env(self, env_vars: t.Dict[str, t.Union[str, int, float, bool]]) -> N self.env_vars[env] = str(val) + def update_launcher_args(self, args: t.Mapping[str, int | str | float | None]) -> None: + self.launcher_args.update(args) + def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: # Store custom arguments in the launcher_args if not isinstance(key, str): raise TypeError("Argument name should be of type str") key = key.strip().lstrip("-") - if key in self._reserved_launch_args: + if key in self.reserved_launch_args: logger.warning( ( f"Could not set argument '{key}': " @@ -400,10 +399,9 @@ def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: self.launcher_args[key] = arg def __str__(self) -> str: # pragma: no-cover - string = "" - string += f"\nLauncher: {self.arg_translator.launcher_str}" + string = f"\nLauncher: {self.arg_translator.launcher_str}" if self.launcher_args: - string += f"\Launch Arguments:\n{fmt_dict(self.launcher_args)}" + string += f"\nLaunch Arguments:\n{fmt_dict(self.launcher_args)}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" return string \ No newline at end of file diff --git a/smartsim/settings/translators/batch/lsf.py b/smartsim/settings/translators/batch/lsf.py index 6a39e5227e..a721217a72 100644 --- a/smartsim/settings/translators/batch/lsf.py +++ b/smartsim/settings/translators/batch/lsf.py @@ -137,9 +137,9 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] prefix = "-" # LSF only uses single dashses - if not value: + if value is None: opts += [prefix + opt] else: - opts += [" ".join((prefix + opt, str(value)))] + opts += [f"{prefix}{opt}", str(value)] return opts \ No newline at end of file diff --git a/smartsim/settings/translators/batch/pbs.py b/smartsim/settings/translators/batch/pbs.py index 4699aa4d34..725376ee13 100644 --- a/smartsim/settings/translators/batch/pbs.py +++ b/smartsim/settings/translators/batch/pbs.py @@ -119,7 +119,7 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] prefix = "-" if not value: raise ValueError("PBS options without values are not allowed") - opts += [" ".join((prefix + opt, str(value)))] + opts += [f"{prefix}{opt}", str(value)] return opts def _sanity_check_resources( @@ -177,9 +177,9 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No if hosts := batch_arg_copy.pop("hostname", None): hosts_list = ["=".join(str(hosts))] select_command += f":{'+'.join(hosts_list)}" - res += [select_command] + res += select_command.split() if walltime := batch_arg_copy.pop("walltime", None): - res += [f"-l walltime={walltime}"] + res += ["-l", f"walltime={walltime}"] # # All other "standard" resource specs # for resource, value in batch_arg_copy.items(): diff --git a/smartsim/settings/translators/batch/slurm.py b/smartsim/settings/translators/batch/slurm.py index fc97e246ab..03e704bdad 100644 --- a/smartsim/settings/translators/batch/slurm.py +++ b/smartsim/settings/translators/batch/slurm.py @@ -125,7 +125,7 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] # TODO add restricted here for opt, value in batch_args.items(): # attach "-" prefix if argument is 1 character otherwise "--" - short_arg = bool(len(str(opt)) == 1) + short_arg = len(opt) == 1 prefix = "-" if short_arg else "--" if not value: diff --git a/smartsim/settings/translators/launch/__init__.py b/smartsim/settings/translators/launch/__init__.py index 3eba5e05db..8deaa227cf 100644 --- a/smartsim/settings/translators/launch/__init__.py +++ b/smartsim/settings/translators/launch/__init__.py @@ -6,14 +6,14 @@ from .pals import PalsMpiexecArgTranslator from .slurm import SlurmArgTranslator -__all__ = [ - "AprunArgTranslator", - "DragonArgTranslator", - "LocalArgTranslator", - "JsrunArgTranslator", - "MpiArgTranslator", - "MpiexecArgTranslator", - "OrteArgTranslator", - "PalsMpiexecArgTranslator", - "SlurmArgTranslator", -] \ No newline at end of file +# __all__ = [ +# "AprunArgTranslator", +# "DragonArgTranslator", +# "LocalArgTranslator", +# "JsrunArgTranslator", +# "MpiArgTranslator", +# "MpiexecArgTranslator", +# "OrteArgTranslator", +# "PalsMpiexecArgTranslator", +# "SlurmArgTranslator", +# ] \ No newline at end of file diff --git a/smartsim/settings/translators/launch/alps.py b/smartsim/settings/translators/launch/alps.py index 6d39c2317f..7b8944ecbf 100644 --- a/smartsim/settings/translators/launch/alps.py +++ b/smartsim/settings/translators/launch/alps.py @@ -40,7 +40,7 @@ def launcher_str(self) -> str: """ return LauncherType.AlpsLauncher.value - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return set() diff --git a/smartsim/settings/translators/launch/dragon.py b/smartsim/settings/translators/launch/dragon.py index d98372b894..c22241d654 100644 --- a/smartsim/settings/translators/launch/dragon.py +++ b/smartsim/settings/translators/launch/dragon.py @@ -41,7 +41,7 @@ def launcher_str(self) -> str: """ return LauncherType.DragonLauncher.value - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return set() diff --git a/smartsim/settings/translators/launch/local.py b/smartsim/settings/translators/launch/local.py index 72bc60134c..48d21999eb 100644 --- a/smartsim/settings/translators/launch/local.py +++ b/smartsim/settings/translators/launch/local.py @@ -41,7 +41,7 @@ def launcher_str(self) -> str: """ return LauncherType.LocalLauncher.value - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return set() diff --git a/smartsim/settings/translators/launch/lsf.py b/smartsim/settings/translators/launch/lsf.py index ae95f0961c..d0095232c6 100644 --- a/smartsim/settings/translators/launch/lsf.py +++ b/smartsim/settings/translators/launch/lsf.py @@ -41,10 +41,10 @@ def launcher_str(self) -> str: """ return LauncherType.LsfLauncher.value - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ - return {"chdir", "h"} + return {"chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"} def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: """Set the number of tasks for this job @@ -87,11 +87,12 @@ def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float, # args launcher uses args = [] restricted = ["chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"] + for opt, value in launcher_args.items(): if opt not in restricted: short_arg = bool(len(str(opt)) == 1) prefix = "-" if short_arg else "--" - if not value: + if value is None: args += [prefix + opt] else: if short_arg: diff --git a/smartsim/settings/translators/launch/mpi.py b/smartsim/settings/translators/launch/mpi.py index 2b1a932c08..a69389c10e 100644 --- a/smartsim/settings/translators/launch/mpi.py +++ b/smartsim/settings/translators/launch/mpi.py @@ -36,7 +36,7 @@ class _BaseMPIArgTranslator(LaunchArgTranslator): - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return {"wd", "wdir"} diff --git a/smartsim/settings/translators/launch/pals.py b/smartsim/settings/translators/launch/pals.py index e59f70870f..5527059514 100644 --- a/smartsim/settings/translators/launch/pals.py +++ b/smartsim/settings/translators/launch/pals.py @@ -41,7 +41,7 @@ def launcher_str(self) -> str: """ return LauncherType.PalsLauncher.value - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return set() diff --git a/smartsim/settings/translators/launch/slurm.py b/smartsim/settings/translators/launch/slurm.py index e1a9b2d719..5297caf096 100644 --- a/smartsim/settings/translators/launch/slurm.py +++ b/smartsim/settings/translators/launch/slurm.py @@ -44,7 +44,7 @@ def launcher_str(self) -> str: """ return LauncherType.SlurmLauncher.value - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return {"chdir", "D"} diff --git a/smartsim/settings/translators/launchArgTranslator.py b/smartsim/settings/translators/launchArgTranslator.py index ea3d18d7d2..d56ff7d621 100644 --- a/smartsim/settings/translators/launchArgTranslator.py +++ b/smartsim/settings/translators/launchArgTranslator.py @@ -50,7 +50,7 @@ def launcher_str(self) -> str: pass @abstractmethod - def _set_reserved_launch_args(self) -> set[str]: + def set_reserved_launch_args(self) -> set[str]: """ Reserved launch keys per Launcher. """ pass diff --git a/smartsim/wlm/slurm.py b/smartsim/wlm/slurm.py index ae7299f28b..63d6afe6ee 100644 --- a/smartsim/wlm/slurm.py +++ b/smartsim/wlm/slurm.py @@ -38,7 +38,9 @@ SSReservedKeywordError, ) from ..log import get_logger -from ..settings.slurmSettings import fmt_walltime +# from ..settings.slurmSettings import fmt_walltime +# Mock function +def fmt_walltime() -> None: ... logger = get_logger(__name__) diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 61b748c0fa..507d154d68 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from smartsim._core.commands.command import Command -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings.launchCommand import LauncherType def test_command_init(): cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 6b62dcee93..945c226cfe 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -26,7 +26,7 @@ from smartsim._core.commands.commandList import CommandList from smartsim._core.commands.command import Command -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings.launchCommand import LauncherType salloc_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) srun_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["srun", "-n", "1"]) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index ff1213499d..2f83311f2d 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,7 +1,7 @@ import pytest -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.alps import AprunArgTranslator -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.alps import AprunArgTranslator +from smartsim.settings.launchCommand import LauncherType import logging def test_launcher_str(): @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert alpsLauncher._reserved_launch_args == set() + assert alpsLauncher.reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", @@ -35,14 +35,12 @@ def test_set_reserved_launcher_args(): ) def test_alps_class_methods(function, value, flag, result): alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert alpsLauncher.launcher.value == LauncherType.AlpsLauncher.value assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) getattr(alpsLauncher, function)(*value) assert alpsLauncher.launcher_args[flag] == result def test_set_verbose_launch(): alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert alpsLauncher.launcher.value == LauncherType.AlpsLauncher.value assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) alpsLauncher.set_verbose_launch(True) assert alpsLauncher.launcher_args == {'debug': 7} @@ -51,7 +49,6 @@ def test_set_verbose_launch(): def test_set_quiet_launch(): aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert aprunLauncher.launcher.value == LauncherType.AlpsLauncher.value assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) aprunLauncher.set_quiet_launch(True) assert aprunLauncher.launcher_args == {'quiet': None} @@ -61,7 +58,7 @@ def test_set_quiet_launch(): def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher, env_vars=env_vars) - assert aprunLauncher.launcher.value == LauncherType.AlpsLauncher.value + assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) aprunLauncher.update_env({"OMP_NUM_THREADS": "10"}) formatted = aprunLauncher.format_env_vars() result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] @@ -111,7 +108,7 @@ def test_invalid_exclude_hostlist_format(): ) def test_unimplimented_methods_throw_warning(caplog, method, params): """Test methods not implemented throw warnings""" - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True @@ -127,7 +124,7 @@ def test_unimplimented_methods_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "aprun") in rec.msg + and (method and "not supported" and "alps") in rec.msg ): break else: diff --git a/tests/temp_tests/test_settings/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py index 0c146f0482..d4b89a8f13 100644 --- a/tests/temp_tests/test_settings/test_batchSettings.py +++ b/tests/temp_tests/test_settings/test_batchSettings.py @@ -1,17 +1,29 @@ -from smartsim.settingshold import BatchSettings -from smartsim.settingshold.batchCommand import SchedulerType +from smartsim.settings import BatchSettings +from smartsim.settings.batchCommand import SchedulerType import pytest +@pytest.mark.parametrize( + "str,scheduler", + [ + pytest.param("slurm", SchedulerType.SlurmScheduler, id="slurm"), + pytest.param("pbs", SchedulerType.PbsScheduler, id="dragon"), + pytest.param("lsf", SchedulerType.LsfScheduler, id="lsf"), + ], +) +def test_create_from_scheduler_str(str, scheduler): + batchSettings = BatchSettings(batch_scheduler=str) + assert batchSettings.batch_scheduler == scheduler + def test_incorrect_env_var_type(): with pytest.raises(TypeError): - _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, env_vars={"str": 2}) + _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, env_vars={"str": 2}) with pytest.raises(TypeError): - _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, env_vars={"str": 2.0}) + _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, env_vars={"str": 2.0}) with pytest.raises(TypeError): - _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, env_vars={"str": "str", "str": 2.0}) + _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, env_vars={"str": "str", "str": 2.0}) def test_incorrect_scheduler_arg_type(): with pytest.raises(TypeError): - _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, launcher_args={"str": [1,2]}) + _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args={"str": [1,2]}) with pytest.raises(TypeError): - _ = BatchSettings(launcher=SchedulerType.SlurmScheduler, launcher_args={"str": SchedulerType.SlurmScheduler}) \ No newline at end of file + _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args={"str": SchedulerType.SlurmScheduler}) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 9f582d00e6..43e803ef1a 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,6 +1,6 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.dragon import DragonArgTranslator -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.dragon import DragonArgTranslator +from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert dragonLauncher._reserved_launch_args == set() + assert dragonLauncher.reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", @@ -23,7 +23,6 @@ def test_set_reserved_launcher_args(): ) def test_dragon_class_methods(function, value, flag, result): dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert dragonLauncher.launcher.value == LauncherType.DragonLauncher.value assert isinstance(dragonLauncher.arg_translator,DragonArgTranslator) getattr(dragonLauncher, function)(*value) assert dragonLauncher.launcher_args[flag] == result @@ -55,7 +54,7 @@ def test_dragon_class_methods(function, value, flag, result): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True diff --git a/tests/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py index 2491b539ec..9f774fe1f4 100644 --- a/tests/temp_tests/test_settings/test_launchSettings.py +++ b/tests/temp_tests/test_settings/test_launchSettings.py @@ -1,8 +1,26 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.launchCommand import LauncherType import pytest import logging +@pytest.mark.parametrize( + "str,launcher", + [ + pytest.param("slurm", LauncherType.SlurmLauncher, id="slrum"), + pytest.param("dragon", LauncherType.DragonLauncher, id="dragon"), + pytest.param("pals", LauncherType.PalsLauncher, id="pals"), + pytest.param("alps", LauncherType.AlpsLauncher,id="alps"), + pytest.param("local", LauncherType.LocalLauncher,id="local"), + pytest.param("mpiexec", LauncherType.MpiexecLauncher,id="mpiexec"), + pytest.param("mpirun", LauncherType.MpirunLauncher,id="mpirun"), + pytest.param("orterun", LauncherType.OrterunLauncher,id="orterun"), + pytest.param("lsf", LauncherType.LsfLauncher,id="lsf"), + ], +) +def test_create_from_launcher_str(str, launcher): + launchSettings = LaunchSettings(launcher=str) + assert launchSettings.launcher == launcher + def test_set_launch_args(): launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) launchSettings.set("str", "some-string") @@ -65,7 +83,7 @@ def test_incorrect_launch_arg_type(): ) def test_prevent_set_reserved_launch_args(caplog, launcher, key): """Test methods not implemented throw warnings""" - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True @@ -94,7 +112,7 @@ def test_prevent_set_reserved_launch_args(caplog, launcher, key): def test_log_overwrite_set_warning_message(caplog): """Test methods not implemented throw warnings""" - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 09365cfd79..403519f40e 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,6 +1,6 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.local import LocalArgTranslator -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.local import LocalArgTranslator +from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) - assert localLauncher._reserved_launch_args == set() + assert localLauncher.reserved_launch_args == set() def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization @@ -104,7 +104,6 @@ def test_format_env_vars(): "D": "12", } localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=env_vars) - assert localLauncher.launcher.value == LauncherType.LocalLauncher.value assert isinstance(localLauncher.arg_translator, LocalArgTranslator) assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] @@ -135,7 +134,7 @@ def test_format_env_vars(): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 9b90649166..c8285a146a 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,6 +1,6 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.lsf import JsrunArgTranslator -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.lsf import JsrunArgTranslator +from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - assert lsfLauncher._reserved_launch_args == {"chdir", "h"} + assert lsfLauncher.reserved_launch_args == {"chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"} @pytest.mark.parametrize( "function,value,result,flag", @@ -23,7 +23,6 @@ def test_set_reserved_launcher_args(): ) def test_lsf_class_methods(function, value, flag, result): lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - assert lsfLauncher.launcher.value == LauncherType.LsfLauncher.value assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) getattr(lsfLauncher, function)(*value) assert lsfLauncher.launcher_args[flag] == result @@ -31,7 +30,6 @@ def test_lsf_class_methods(function, value, flag, result): def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, env_vars=env_vars) - assert lsfLauncher.launcher.value == LauncherType.LsfLauncher.value assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) formatted = lsfLauncher.format_env_vars() assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] @@ -46,7 +44,7 @@ def test_launch_args(): "np": 100, } lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, launcher_args=launch_args) - assert lsfLauncher.launcher.value == "jsrun" + assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) formatted = lsfLauncher.format_launcher_args() result = [ "--latency_priority=gpu-gpu", @@ -85,7 +83,7 @@ def test_launch_args(): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True @@ -101,7 +99,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "jsrun") in rec.msg + and (method and "not supported" and "lsf") in rec.msg ): break else: diff --git a/tests/temp_tests/test_settings/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py index a5afaf72bc..a541e66b6f 100644 --- a/tests/temp_tests/test_settings/test_lsfScheduler.py +++ b/tests/temp_tests/test_settings/test_lsfScheduler.py @@ -1,11 +1,11 @@ -from smartsim.settingshold import BatchSettings +from smartsim.settings import BatchSettings import pytest import logging -from smartsim.settingshold.batchCommand import SchedulerType +from smartsim.settings.batchCommand import SchedulerType def test_scheduler_str(): """Ensure launcher_str returns appropriate value""" - lsfScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) + lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) assert lsfScheduler.scheduler_str() == SchedulerType.LsfScheduler.value @pytest.mark.parametrize( @@ -23,18 +23,18 @@ def test_scheduler_str(): ], ) def test_update_env_initialized(function, value, flag, result): - lsfScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) + lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) getattr(lsfScheduler, function)(*value) assert lsfScheduler.scheduler_args[flag] == result def test_create_bsub(): batch_args = {"core_isolation": None} - lsfScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler, scheduler_args=batch_args) + lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler, scheduler_args=batch_args) lsfScheduler.set_nodes(1) lsfScheduler.set_walltime("10:10:10") lsfScheduler.set_queue("default") args = lsfScheduler.format_batch_args() - assert args == ["-core_isolation", "-nnodes 1", "-W 10:10", "-q default"] + assert args == ["-core_isolation", "-nnodes", "1", "-W", "10:10", "-q", "default"] @pytest.mark.parametrize( "method,params", @@ -45,14 +45,14 @@ def test_create_bsub(): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True with caplog.at_level(logging.WARNING): caplog.clear() - slurmScheduler = BatchSettings(scheduler=SchedulerType.LsfScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) try: getattr(slurmScheduler, method)(*params) finally: @@ -61,7 +61,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "bsub") in rec.msg + and (method and "not supported" and "lsf") in rec.msg ): break else: diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 7c1615706b..3e61046939 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -1,9 +1,9 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator import pytest import logging import itertools -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings.launchCommand import LauncherType @pytest.mark.parametrize( "launcher", @@ -29,7 +29,7 @@ def test_launcher_str(launcher): def test_set_reserved_launcher_args(launcher): """Ensure launcher_str returns appropriate value""" mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings._reserved_launch_args == {"wd", "wdir"} + assert mpiSettings.reserved_launch_args == {"wd", "wdir"} @pytest.mark.parametrize( "l,function,value,result,flag", @@ -56,7 +56,6 @@ def test_set_reserved_launcher_args(launcher): def test_mpi_class_methods(l,function, value, flag, result): mpiSettings = LaunchSettings(launcher=l[0]) assert isinstance(mpiSettings.arg_translator,l[1]) - assert mpiSettings.launcher.value == l[0].value getattr(mpiSettings, function)(*value) assert mpiSettings.launcher_args[flag] == result @@ -71,7 +70,6 @@ def test_mpi_class_methods(l,function, value, flag, result): def test_format_env_vars(launcher): env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} mpiSettings = LaunchSettings(launcher=launcher, env_vars=env_vars) - assert mpiSettings.launcher.value == launcher.value formatted = mpiSettings.format_env_vars() result = [ "-x", @@ -108,7 +106,6 @@ def test_format_launcher_args(launcher): ) def test_set_verbose_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings.launcher.value == launcher.value mpiSettings.set_verbose_launch(True) assert mpiSettings.launcher_args == {'verbose': None} mpiSettings.set_verbose_launch(False) @@ -124,7 +121,6 @@ def test_set_verbose_launch(launcher): ) def test_set_quiet_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings.launcher.value == launcher.value mpiSettings.set_quiet_launch(True) assert mpiSettings.launcher_args == {'quiet': None} mpiSettings.set_quiet_launch(False) @@ -180,7 +176,7 @@ def test_launcher_str(launcher): ], ) def test_unimplimented_methods_throw_warning(l, caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 904c3c54fb..f8d2b9b71a 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -1,6 +1,6 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.pals import PalsMpiexecArgTranslator -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.pals import PalsMpiexecArgTranslator +from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) - assert palsLauncher._reserved_launch_args == set() + assert palsLauncher.reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", @@ -28,7 +28,6 @@ def test_set_reserved_launcher_args(): def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) getattr(palsLauncher, function)(*value) - assert palsLauncher.launcher == LauncherType.PalsLauncher assert isinstance(palsLauncher.arg_translator,PalsMpiexecArgTranslator) assert palsLauncher.launcher_args[flag] == result assert palsLauncher.format_launcher_args() == ["--" + flag, str(result)] @@ -71,7 +70,7 @@ def test_invalid_hostlist_format(): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index 614ec181ce..ce307806de 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -1,12 +1,12 @@ -from smartsim.settingshold import BatchSettings -from smartsim.settingshold.translators.batch.pbs import QsubBatchArgTranslator -from smartsim.settingshold.batchCommand import SchedulerType +from smartsim.settings import BatchSettings +from smartsim.settings.translators.batch.pbs import QsubBatchArgTranslator +from smartsim.settings.batchCommand import SchedulerType import pytest import logging def test_scheduler_str(): """Ensure launcher_str returns appropriate value""" - pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) + pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) assert pbsScheduler.scheduler_str() == SchedulerType.PbsScheduler.value @pytest.mark.parametrize( @@ -22,24 +22,23 @@ def test_scheduler_str(): ], ) def test_update_env_initialized(function, value, flag, result): - pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) + pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) getattr(pbsScheduler, function)(*value) assert pbsScheduler.scheduler_args[flag] == result def test_create_pbs_batch(): - pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) + pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) pbsScheduler.set_nodes(1) pbsScheduler.set_walltime("10:00:00") pbsScheduler.set_queue("default") pbsScheduler.set_account("myproject") pbsScheduler.set_ncpus(10) args = pbsScheduler.format_batch_args() - print(f"here: {args}") assert args == [ - "-l nodes=1:ncpus=10", - "-l walltime=10:00:00", - "-q default", - "-A myproject", + "-l", "nodes=1:ncpus=10", + "-l", "walltime=10:00:00", + "-q", "default", + "-A", "myproject", ] @pytest.mark.parametrize( @@ -53,14 +52,14 @@ def test_create_pbs_batch(): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True with caplog.at_level(logging.WARNING): caplog.clear() - pbsScheduler = BatchSettings(scheduler=SchedulerType.PbsScheduler) + pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) try: getattr(pbsScheduler, method)(*params) finally: @@ -69,7 +68,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "qsub") in rec.msg + and (method and "not supported" and "pbs") in rec.msg ): break else: diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 50c726bc37..76778f19c1 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,6 +1,6 @@ -from smartsim.settingshold import LaunchSettings -from smartsim.settingshold.translators.launch.slurm import SlurmArgTranslator -from smartsim.settingshold.launchCommand import LauncherType +from smartsim.settings import LaunchSettings +from smartsim.settings.translators.launch.slurm import SlurmArgTranslator +from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -12,7 +12,7 @@ def test_launcher_str(): def test_set_reserved_launcher_args(): """Ensure launcher_str returns appropriate value""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert slurmLauncher._reserved_launch_args == {"chdir", "D"} + assert slurmLauncher.reserved_launch_args == {"chdir", "D"} @pytest.mark.parametrize( "function,value,result,flag", @@ -37,14 +37,12 @@ def test_set_reserved_launcher_args(): ) def test_slurm_class_methods(function, value, flag, result): slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) getattr(slurmLauncher, function)(*value) assert slurmLauncher.launcher_args[flag] == result def test_set_verbose_launch(): slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) slurmLauncher.set_verbose_launch(True) assert slurmLauncher.launcher_args == {'verbose': None} @@ -53,7 +51,6 @@ def test_set_verbose_launch(): def test_set_quiet_launch(): slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) slurmLauncher.set_quiet_launch(True) assert slurmLauncher.launcher_args == {'quiet': None} @@ -68,7 +65,6 @@ def test_format_env_vars(): "SSKEYIN": "name_0,name_1", } slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) - assert slurmLauncher.launcher.value == LauncherType.SlurmLauncher.value assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) formatted = slurmLauncher.format_env_vars() assert "OMP_NUM_THREADS=20" in formatted @@ -200,7 +196,7 @@ def test_invalid_walltime_format(): ) def test_unimplimented_methods_throw_warning(caplog, method, params): """Test methods not implemented throw warnings""" - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index ddd7005fe2..1c53552ad2 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -1,12 +1,12 @@ -from smartsim.settingshold import BatchSettings -from smartsim.settingshold.translators.batch.slurm import SlurmBatchArgTranslator -from smartsim.settingshold.batchCommand import SchedulerType +from smartsim.settings import BatchSettings +from smartsim.settings.translators.batch.slurm import SlurmBatchArgTranslator +from smartsim.settings.batchCommand import SchedulerType import pytest import logging def test_scheduler_str(): """Ensure launcher_str returns appropriate value""" - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) assert slurmScheduler.scheduler_str() == SchedulerType.SlurmScheduler.value @pytest.mark.parametrize( @@ -23,13 +23,13 @@ def test_scheduler_str(): ], ) def test_update_env_initialized(function, value, flag, result): - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) getattr(slurmScheduler, function)(*value) assert slurmScheduler.scheduler_args[flag] == result def test_create_sbatch(): batch_args = {"exclusive": None, "oversubscribe": None} - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler, scheduler_args=batch_args) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args=batch_args) assert isinstance(slurmScheduler.arg_translator, SlurmBatchArgTranslator) #assert slurmScheduler.batch_args["partition"] == "default" args = slurmScheduler.format_batch_args() @@ -48,7 +48,7 @@ def test_launch_args_input_mutation(): key1: val1, key2: val2, } - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler, scheduler_args=default_scheduler_args) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args=default_scheduler_args) # Confirm initial values are set assert slurmScheduler.scheduler_args[key0] == val0 @@ -64,14 +64,14 @@ def test_launch_args_input_mutation(): def test_sbatch_settings(): scheduler_args = {"nodes": 1, "time": "10:00:00", "account": "A3123"} - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler,scheduler_args=scheduler_args) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler,scheduler_args=scheduler_args) formatted = slurmScheduler.format_batch_args() result = ["--nodes=1", "--time=10:00:00", "--account=A3123"] assert formatted == result def test_sbatch_manual(): - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) slurmScheduler.set_nodes(5) slurmScheduler.set_account("A3531") slurmScheduler.set_walltime("10:00:00") @@ -89,14 +89,14 @@ def test_sbatch_manual(): ], ) def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.base import logger + from smartsim.settings.launchSettings import logger prev_prop = logger.propagate logger.propagate = True with caplog.at_level(logging.WARNING): caplog.clear() - slurmScheduler = BatchSettings(scheduler=SchedulerType.SlurmScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) try: getattr(slurmScheduler, method)(*params) finally: @@ -105,7 +105,7 @@ def test_unimplimented_setters_throw_warning(caplog, method, params): for rec in caplog.records: if ( logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "sbatch") in rec.msg + and (method and "not supported" and "slurm") in rec.msg ): break else: From bdce4bd8ff23220d3341a6c86726d2f81a183f48 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 29 May 2024 16:41:20 -0500 Subject: [PATCH 26/43] launch commands --- smartsim/_core/commands/__init__.py | 3 +- smartsim/_core/commands/launchCommands.py | 50 +++++++++++++++++++ .../test_commands/test_launchCommands.py | 17 +++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 smartsim/_core/commands/launchCommands.py create mode 100644 tests/temp_tests/test_core/test_commands/test_launchCommands.py diff --git a/smartsim/_core/commands/__init__.py b/smartsim/_core/commands/__init__.py index 670e3e06b5..739f8563b3 100644 --- a/smartsim/_core/commands/__init__.py +++ b/smartsim/_core/commands/__init__.py @@ -25,4 +25,5 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from .command import Command -from .commandList import CommandList \ No newline at end of file +from .commandList import CommandList +from .launchCommands import LaunchCommands \ No newline at end of file diff --git a/smartsim/_core/commands/launchCommands.py b/smartsim/_core/commands/launchCommands.py new file mode 100644 index 0000000000..26b622385c --- /dev/null +++ b/smartsim/_core/commands/launchCommands.py @@ -0,0 +1,50 @@ +from .commandList import CommandList + +class LaunchCommands: + """Container or aggregating prelaunch commands (e.g. file + system operations), launch commands, and postlaunch commands + """ + def __init__( + self, + prelaunch_commands: CommandList, + launch_commands: CommandList, + postlaunch_commands: CommandList, + ) -> None: + """LaunchCommand constructor + """ + self._prelaunch_commands = prelaunch_commands + self._launch_commands = launch_commands + self._postlaunch_commands = postlaunch_commands + + @property + def prelaunch_command_maps(self) -> CommandList: + """Get the prelaunch command list. + Return a reference to the command list. + """ + return self._prelaunch_commands + + @property + def launch_command_maps(self) -> CommandList: + """Get the launch command list. + Return a reference to the command list. + """ + return self._launch_commands + + @property + def postlaunch_command_maps(self) -> CommandList: + """Get the postlaunch command list. + Return a reference to the command list. + """ + return self._postlaunch_commands + + def __str__(self) -> str: # pragma: no cover + string = "\n\nPrelaunch Command List:\n" + for _, pre_cmd in enumerate(self.prelaunch_command_maps): + string += f"{pre_cmd}\n" + string += "\n\nLaunch Command List:\n" + for _, launch_cmd in enumerate(self.launch_command_maps): + string += f"{launch_cmd}\n" + string += "\n\nPostlaunch Command List:\n" + for _, post_cmd in enumerate(self.postlaunch_command_maps): + string += f"{post_cmd}\n" + return string \ No newline at end of file diff --git a/tests/temp_tests/test_core/test_commands/test_launchCommands.py b/tests/temp_tests/test_core/test_commands/test_launchCommands.py new file mode 100644 index 0000000000..26cfbca1f9 --- /dev/null +++ b/tests/temp_tests/test_core/test_commands/test_launchCommands.py @@ -0,0 +1,17 @@ +from smartsim._core.commands.commandList import CommandList +from smartsim._core.commands.command import Command +from smartsim._core.commands.launchCommands import LaunchCommands +from smartsim.settings.launchCommand import LauncherType + +pre_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["pre", "cmd"]) +launch_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["launch", "cmd"]) +post_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["post", "cmd"]) +pre_commands_list = CommandList(commands=[pre_cmd]) +launch_command_list = CommandList(commands=[launch_cmd]) +post_command_list = CommandList(commands=[post_cmd]) + +def test_launchCommand_init(): + launch_cmd = LaunchCommands(prelaunch_commands=pre_commands_list,launch_commands=launch_command_list,postlaunch_commands=post_command_list) + assert launch_cmd.prelaunch_command_maps == pre_commands_list + assert launch_cmd.launch_command_maps == launch_command_list + assert launch_cmd.postlaunch_command_maps == post_command_list From 60cad541bf894f5a91ace3ee7c303142634cfa28 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 5 Jun 2024 01:55:50 -0500 Subject: [PATCH 27/43] chnages --- smartsim/_core/commands/launchCommands.py | 2 +- smartsim/_core/launcher/__init__.py | 2 +- .../_core/launcher/dragon/dragonLauncher.py | 22 +- smartsim/_core/launcher/local/local.py | 5 +- smartsim/_core/launcher/lsf/lsfLauncher.py | 26 +- smartsim/_core/launcher/pbs/pbsLauncher.py | 29 +- .../_core/launcher/slurm/slurmLauncher.py | 26 +- smartsim/_core/launcher/step/alpsStep.py | 10 +- smartsim/_core/launcher/step/dragonStep.py | 17 +- smartsim/_core/launcher/step/localStep.py | 8 +- smartsim/_core/launcher/step/lsfStep.py | 8 +- smartsim/_core/launcher/step/mpiStep.py | 9 +- smartsim/settings/__init__.py | 17 +- smartsim/settings/batchSettings.py | 193 ++------- smartsim/settings/common.py | 23 +- smartsim/settings/launchSettings.py | 375 ++++-------------- smartsim/settings/translators/batch/lsf.py | 49 ++- smartsim/settings/translators/batch/pbs.py | 57 +-- smartsim/settings/translators/batch/slurm.py | 48 ++- .../translators/batchArgTranslator.py | 80 +--- .../settings/translators/launch/__init__.py | 22 +- smartsim/settings/translators/launch/alps.py | 106 +++-- .../settings/translators/launch/dragon.py | 35 +- smartsim/settings/translators/launch/local.py | 33 +- smartsim/settings/translators/launch/lsf.py | 65 +-- smartsim/settings/translators/launch/mpi.py | 125 +++--- smartsim/settings/translators/launch/pals.py | 91 ++--- smartsim/settings/translators/launch/slurm.py | 118 +++--- .../translators/launchArgTranslator.py | 182 +-------- .../test_settings/test_alpsLauncher.py | 109 ++--- .../test_settings/test_batchSettings.py | 45 ++- .../test_settings/test_dragonLauncher.py | 74 +--- .../test_settings/test_launchSettings.py | 258 ++++++------ .../test_settings/test_localLauncher.py | 123 ++---- .../test_settings/test_lsfLauncher.py | 83 +--- .../test_settings/test_lsfScheduler.py | 59 +-- .../test_settings/test_mpiLauncher.py | 127 +----- .../test_settings/test_palsLauncher.py | 76 +--- .../test_settings/test_pbsScheduler.py | 64 +-- .../test_settings/test_slurmLauncher.py | 151 +++---- .../test_settings/test_slurmScheduler.py | 72 +--- 41 files changed, 958 insertions(+), 2066 deletions(-) diff --git a/smartsim/_core/commands/launchCommands.py b/smartsim/_core/commands/launchCommands.py index 26b622385c..daeada7534 100644 --- a/smartsim/_core/commands/launchCommands.py +++ b/smartsim/_core/commands/launchCommands.py @@ -1,7 +1,7 @@ from .commandList import CommandList class LaunchCommands: - """Container or aggregating prelaunch commands (e.g. file + """Container for aggregating prelaunch commands (e.g. file system operations), launch commands, and postlaunch commands """ def __init__( diff --git a/smartsim/_core/launcher/__init__.py b/smartsim/_core/launcher/__init__.py index d789096415..a50a7c3ce8 100644 --- a/smartsim/_core/launcher/__init__.py +++ b/smartsim/_core/launcher/__init__.py @@ -38,4 +38,4 @@ "LSFLauncher", "PBSLauncher", "SlurmLauncher", -] +] \ No newline at end of file diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index f4f89974c2..fee78160c6 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -32,19 +32,13 @@ from ...._core.launcher.stepMapping import StepMap from ....error import LauncherError, SmartSimError from ....log import get_logger -# from ....settings import ( -# DragonRunSettings, -# QsubBatchSettings, -# RunSettings, -# SbatchSettings, -# SettingsBase, -# ) -# Mock imports for now -class DragonRunSettings: pass -class QsubBatchSettings: pass -class RunSettings: pass -class SbatchSettings: pass -class SettingsBase: pass +from ....settings import ( + DragonRunSettings, + QsubBatchSettings, + RunSettings, + SbatchSettings, + SettingsBase, +) from ....status import SmartSimStatus from ...schemas import ( @@ -98,7 +92,7 @@ def cleanup(self) -> None: # RunSettings types supported by this launcher # t.Type[SettingsBase] @property - def supported_rs(self) -> t.Dict[t.Type[Step]]: + def supported_rs(self) -> t.Dict[t.Type[SettingsBase], t.Type[Step]]: # RunSettings types supported by this launcher pass # return { diff --git a/smartsim/_core/launcher/local/local.py b/smartsim/_core/launcher/local/local.py index fd47737ad8..2b3f479971 100644 --- a/smartsim/_core/launcher/local/local.py +++ b/smartsim/_core/launcher/local/local.py @@ -27,10 +27,7 @@ import typing as t from ....entity import SmartSimEntity -# from ....settings import RunSettings, SettingsBase -# Mock imports -class RunSettings: pass -class SettingsBase: pass +from ....settings import RunSettings, SettingsBase from ..launcher import Launcher from ..step import LocalStep, Step from ..stepInfo import StepInfo, UnmanagedStepInfo diff --git a/smartsim/_core/launcher/lsf/lsfLauncher.py b/smartsim/_core/launcher/lsf/lsfLauncher.py index 027b5e4e66..d91f969e3e 100644 --- a/smartsim/_core/launcher/lsf/lsfLauncher.py +++ b/smartsim/_core/launcher/lsf/lsfLauncher.py @@ -29,23 +29,15 @@ from ....error import LauncherError from ....log import get_logger -# from ....settings import ( -# BsubBatchSettings, -# JsrunSettings, -# MpiexecSettings, -# MpirunSettings, -# OrterunSettings, -# RunSettings, -# SettingsBase, -# ) -# Mock imports -class BsubBatchSettings: pass -class JsrunSettings: pass -class MpiexecSettings: pass -class MpirunSettings: pass -class OrterunSettings: pass -class RunSettings: pass -class SettingsBase: pass +from ....settings import ( + BsubBatchSettings, + JsrunSettings, + MpiexecSettings, + MpirunSettings, + OrterunSettings, + RunSettings, + SettingsBase, +) from ....status import SmartSimStatus from ...config import CONFIG diff --git a/smartsim/_core/launcher/pbs/pbsLauncher.py b/smartsim/_core/launcher/pbs/pbsLauncher.py index d870d40473..db6a768f51 100644 --- a/smartsim/_core/launcher/pbs/pbsLauncher.py +++ b/smartsim/_core/launcher/pbs/pbsLauncher.py @@ -29,25 +29,16 @@ from ....error import LauncherError from ....log import get_logger -# from ....settings import ( -# AprunSettings, -# MpiexecSettings, -# MpirunSettings, -# OrterunSettings, -# PalsMpiexecSettings, -# QsubBatchSettings, -# RunSettings, -# SettingsBase, -# ) -# Mock imports -class AprunSettings: pass -class MpiexecSettings: pass -class MpirunSettings: pass -class OrterunSettings: pass -class PalsMpiexecSettings: pass -class QsubBatchSettings: pass -class RunSettings: pass -class SettingsBase: pass +from ....settings import ( + AprunSettings, + MpiexecSettings, + MpirunSettings, + OrterunSettings, + PalsMpiexecSettings, + QsubBatchSettings, + RunSettings, + SettingsBase, +) from ....status import SmartSimStatus from ...config import CONFIG diff --git a/smartsim/_core/launcher/slurm/slurmLauncher.py b/smartsim/_core/launcher/slurm/slurmLauncher.py index fca108ff01..a1742415c4 100644 --- a/smartsim/_core/launcher/slurm/slurmLauncher.py +++ b/smartsim/_core/launcher/slurm/slurmLauncher.py @@ -31,23 +31,15 @@ from ....error import LauncherError from ....log import get_logger -# from ....settings import ( -# MpiexecSettings, -# MpirunSettings, -# OrterunSettings, -# RunSettings, -# SbatchSettings, -# SettingsBase, -# SrunSettings, -# ) -# Mock imports -class SrunSettings: pass -class MpiexecSettings: pass -class MpirunSettings: pass -class OrterunSettings: pass -class SbatchSettings: pass -class RunSettings: pass -class SettingsBase: pass +from ....settings import ( + MpiexecSettings, + MpirunSettings, + OrterunSettings, + RunSettings, + SbatchSettings, + SettingsBase, + SrunSettings, +) from ....status import SmartSimStatus from ...config import CONFIG diff --git a/smartsim/_core/launcher/step/alpsStep.py b/smartsim/_core/launcher/step/alpsStep.py index 27359852d1..6c6ee89fc7 100644 --- a/smartsim/_core/launcher/step/alpsStep.py +++ b/smartsim/_core/launcher/step/alpsStep.py @@ -32,18 +32,14 @@ from ....entity import DBNode, Model from ....error import AllocationError from ....log import get_logger -# from ....settings import AprunSettings, RunSettings, Singularity -# Mock imports for now -class AprunSettings: pass -class RunSettings: pass -class Singularity: pass +from ....settings import AprunSettings, RunSettings, Singularity from .step import Step, proxyable_launch_cmd logger = get_logger(__name__) class AprunStep(Step): def __init__( - self, entity: t.Union[Model, DBNode], run_settings: str + self, entity: t.Union[Model, DBNode], run_settings: AprunSettings ) -> None: """Initialize a ALPS aprun job step @@ -92,7 +88,7 @@ def get_launch_cmd(self) -> t.List[str]: launch_script_path = self.get_colocated_launch_script() aprun_cmd.extend([bash, launch_script_path]) - if isinstance(self.run_settings.container, str): #changed from Singularity to str + if isinstance(self.run_settings.container, Singularity): #changed from Singularity to str # pylint: disable-next=protected-access aprun_cmd += self.run_settings.container._container_cmds(self.cwd) diff --git a/smartsim/_core/launcher/step/dragonStep.py b/smartsim/_core/launcher/step/dragonStep.py index 75ab0fb78c..6380be5fa6 100644 --- a/smartsim/_core/launcher/step/dragonStep.py +++ b/smartsim/_core/launcher/step/dragonStep.py @@ -33,17 +33,12 @@ from ...._core.schemas.dragonRequests import DragonRunRequest, request_registry from ....error.errors import SSUnsupportedError from ....log import get_logger -# from ....settings import ( -# DragonRunSettings, -# QsubBatchSettings, -# SbatchSettings, -# Singularity, -# ) -# Temp mock imports -class DragonRunSettings: pass -class QsubBatchSettings: pass -class SbatchSettings: pass -class Singularity: pass +from ....settings import ( + DragonRunSettings, + QsubBatchSettings, + SbatchSettings, + Singularity, +) from .step import Step diff --git a/smartsim/_core/launcher/step/localStep.py b/smartsim/_core/launcher/step/localStep.py index 679d61a55d..0df8d0ff9e 100644 --- a/smartsim/_core/launcher/step/localStep.py +++ b/smartsim/_core/launcher/step/localStep.py @@ -29,14 +29,10 @@ import typing as t from ....entity import DBNode, Model -# from ....settings import Singularity -# from ....settings.base import RunSettings +from ....settings import Singularity +from ....settings import RunSettings from .step import Step, proxyable_launch_cmd -# Temperary mock imports -class Singularity: pass -class RunSettings: pass - class LocalStep(Step): def __init__(self, entity: t.Union[Model, DBNode], run_settings: RunSettings): diff --git a/smartsim/_core/launcher/step/lsfStep.py b/smartsim/_core/launcher/step/lsfStep.py index 7a07c852e9..b006a1eb7c 100644 --- a/smartsim/_core/launcher/step/lsfStep.py +++ b/smartsim/_core/launcher/step/lsfStep.py @@ -31,12 +31,8 @@ from ....entity import DBNode, Model from ....error import AllocationError from ....log import get_logger -# from ....settings import BsubBatchSettings, JsrunSettings -# from ....settings.base import RunSettings -# Temp mock imports -class BsubBatchSettings: pass -class JsrunSettings: pass -class RunSettings: pass +from ....settings import BsubBatchSettings, JsrunSettings +from ....settings import RunSettings from .step import Step diff --git a/smartsim/_core/launcher/step/mpiStep.py b/smartsim/_core/launcher/step/mpiStep.py index 746f7e48c0..457bddae6f 100644 --- a/smartsim/_core/launcher/step/mpiStep.py +++ b/smartsim/_core/launcher/step/mpiStep.py @@ -32,13 +32,8 @@ from ....entity import DBNode, Model from ....error import AllocationError, SmartSimError from ....log import get_logger -# from ....settings import MpiexecSettings, MpirunSettings, OrterunSettings -# from ....settings.base import RunSettings -# Temp mock imports -class MpiexecSettings: pass -class MpirunSettings: pass -class OrterunSettings: pass -class RunSettings: pass +from ....settings import MpiexecSettings, MpirunSettings, OrterunSettings +from ....settings import RunSettings from .step import Step, proxyable_launch_cmd diff --git a/smartsim/settings/__init__.py b/smartsim/settings/__init__.py index 41076efd3e..33b54de91a 100644 --- a/smartsim/settings/__init__.py +++ b/smartsim/settings/__init__.py @@ -32,4 +32,19 @@ "LaunchSettings", "BaseSettings", "BatchSettings" -] \ No newline at end of file +] + +class DragonRunSettings: pass +class QsubBatchSettings: pass +class SbatchSettings: pass +class Singularity: pass +class SettingsBase: pass +class AprunSettings: pass +class RunSettings: pass +class OrterunSettings: pass +class MpirunSettings: pass +class MpiexecSettings: pass +class JsrunSettings: pass +class BsubBatchSettings: pass +class PalsMpiexecSettings: pass +class SrunSettings: pass \ No newline at end of file diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 285a983443..9954e87cb9 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -30,7 +30,7 @@ from smartsim.log import get_logger from .._core.utils.helpers import fmt_dict -from .common import validate_env_vars, validate_args, StringArgument +from .common import StringArgument from .batchCommand import SchedulerType from .translators.batch.pbs import QsubBatchArgTranslator from .translators.batch.slurm import SlurmBatchArgTranslator @@ -49,187 +49,54 @@ def __init__( ) -> None: try: self._batch_scheduler = SchedulerType(batch_scheduler) - except KeyError: + except ValueError: raise ValueError(f"Invalid scheduler type: {batch_scheduler}") - self._arg_translator = self._get_scheduler() - - if env_vars: - validate_env_vars(env_vars) + self._arg_translator = self._get_arg_builder(scheduler_args) self.env_vars = env_vars or {} - if scheduler_args: - validate_args(scheduler_args) - self.scheduler_args = scheduler_args or {} - @property - def batch_scheduler(self): - return self._batch_scheduler + def batch_scheduler(self) -> str: + """Return the scheduler name. + """ + return self._batch_scheduler.value @property - def arg_translator(self): + def scheduler_args(self) -> BatchArgTranslator: + """Return the batch argument translator. + """ + # Is a deep copy needed here? return self._arg_translator @property - def scheduler_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: - """Retrieve attached batch arguments - - :returns: attached batch arguments + def env_vars(self) -> StringArgument: + """Return an immutable list of attached environment variables. """ - return self._scheduler_args - - @scheduler_args.setter - def scheduler_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> None: - """Attach batch arguments + return copy.deepcopy(self._env_vars) - :param value: dictionary of batch arguments + @env_vars.setter + def env_vars(self, value: t.Mapping[str, str]) -> None: + """Set the environment variables. """ - self._scheduler_args = copy.deepcopy(value) if value else {} + self._env_vars = copy.deepcopy(value) - def _get_scheduler(self) -> BatchArgTranslator: + def _get_arg_builder(self, scheduler_args) -> BatchArgTranslator: """ Map the Scheduler to the BatchArgTranslator """ - if self._batch_scheduler.value == 'slurm': - return SlurmBatchArgTranslator() - elif self._batch_scheduler.value == 'lsf': - return BsubBatchArgTranslator() - elif self._batch_scheduler.value == 'pbs': - return QsubBatchArgTranslator() - - def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ - return self.arg_translator.scheduler_str() - - def set_walltime(self, walltime: str) -> None: - """Set the walltime of the job - - format = "HH:MM:SS" - - :param walltime: wall time - """ - # TODO check for formatting here - args = self.arg_translator.set_walltime(walltime) - if args: - self.update_scheduler_args(args) - - def set_nodes(self, num_nodes: int) -> None: - """Set the number of nodes for this batch job - - :param num_nodes: number of nodes - """ - args = self.arg_translator.set_nodes(num_nodes) - if args: - self.update_scheduler_args(args) - - def set_account(self, account: str) -> None: - """Set the account for this batch job - - :param account: account id - """ - args = self.arg_translator.set_account(account) - if args: - self.update_scheduler_args(args) - - def set_partition(self, partition: str) -> None: - """Set the partition for the batch job - - :param partition: partition name - """ - args = self.arg_translator.set_partition(partition) - if args: - self.update_scheduler_args(args) + if self._batch_scheduler == SchedulerType.SlurmScheduler: + return SlurmBatchArgTranslator(scheduler_args) + elif self._batch_scheduler == SchedulerType.LsfScheduler: + return BsubBatchArgTranslator(scheduler_args) + elif self._batch_scheduler == SchedulerType.PbsScheduler: + return QsubBatchArgTranslator(scheduler_args) + else: + raise ValueError(f"Invalid scheduler type: {self._batch_scheduler}") - def set_queue(self, queue: str) -> None: - """alias for set_partition - - Sets the partition for the slurm batch job - - :param queue: the partition to run the batch job on - """ - args = self.arg_translator.set_queue(queue) - if args: - self.update_scheduler_args(args) - - def set_cpus_per_task(self, cpus_per_task: int) -> None: - """Set the number of cpus to use per task - - This sets ``--cpus-per-task`` - - :param num_cpus: number of cpus to use per task - """ - args = self.arg_translator.set_cpus_per_task(cpus_per_task) - if args: - self.update_scheduler_args(args) - - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - """Specify the hostlist for this job - - :param host_list: hosts to launch on - :raises TypeError: if not str or list of str - """ - args = self.arg_translator.set_hostlist(host_list) - if args: - self.update_scheduler_args(args) - - def set_smts(self, smts: int) -> None: - """Set SMTs - - This sets ``-alloc_flags``. If the user sets - SMT explicitly through ``-alloc_flags``, then that - takes precedence. - - :param smts: SMT (e.g on Summit: 1, 2, or 4) - """ - args = self.arg_translator.set_smts(smts) - if args: - self.update_scheduler_args(args) - - def set_project(self, project: str) -> None: - """Set the project - - This sets ``-P``. - - :param time: project name - """ - args = self.arg_translator.set_project(project) - if args: - self.update_scheduler_args(args) - - def set_tasks(self, tasks: int) -> None: - """Set the number of tasks for this job - - This sets ``-n`` - - :param tasks: number of tasks - """ - args = self.arg_translator.set_tasks(tasks) - if args: - self.update_scheduler_args(args) - - def set_ncpus(self, num_cpus: int) -> None: - """Set the number of cpus obtained in each node. - - If a select argument is provided in - ``QsubBatchSettings.resources``, then - this value will be overridden - - :param num_cpus: number of cpus per node in select - """ - args = self.arg_translator.set_ncpus(num_cpus) - if args: - self.update_scheduler_args(args) - def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview - """ - return self.arg_translator.format_batch_args(self.scheduler_args) - - def update_scheduler_args(self, args: t.Mapping[str, int | str | float | None]) -> None: - self.scheduler_args.update(args) - def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: - # Store custom arguments in the launcher_args - self.scheduler_args[key] = arg + :return: batch arguments for Sbatch + """ + return self._arg_translator.format_batch_args() def __str__(self) -> str: # pragma: no-cover string = f"\nScheduler: {self.arg_translator.scheduler_str}" diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index e2fa5d658a..8b61d3e7df 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -29,17 +29,14 @@ IntegerArgument = t.Dict[str, t.Optional[int]] StringArgument = t.Dict[str, t.Optional[str]] -def validate_env_vars(env_vars: StringArgument) -> None: - """Validate that user passed env vars are of correct type. - """ - for key, value in env_vars.items(): - if not isinstance(value, str) and value is not None: - raise TypeError(f"Value for '{key}' must be a string.") - -def validate_args(launch_args: t.Mapping[str, t.Union[str,int,float,None]]) -> None: - """Validate that user passed launch args and scheduler args are of correct type. - """ - for key, value in launch_args.items(): - if not isinstance(value, (str,int,float)) and value is not None: - raise TypeError(f"Value for '{key}' must be a string, an int, a float, or None.") +def set_check_input(key: str, value: str | None, logger): + # TODO check this + if not (isinstance(key, str) or isinstance(value, (str, None))): + raise TypeError("Argument name should be of type str") + if key.startswith("-"): + key = key.lstrip("-") + logger.warning( + "One or more leading `-` characters were provided to the run argument. \ + Leading dashes were stripped and the arguments were passed to the run_command." + ) \ No newline at end of file diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 15ea8e9a22..7e39548a58 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -31,7 +31,7 @@ from smartsim.log import get_logger from .._core.utils.helpers import fmt_dict -from .common import validate_env_vars, validate_args, StringArgument +from .common import StringArgument from .launchCommand import LauncherType from .translators.launch.alps import AprunArgTranslator from .translators.launch.lsf import JsrunArgTranslator @@ -49,359 +49,120 @@ class LaunchSettings(BaseSettings): def __init__( self, launcher: t.Union[LauncherType, str], - launcher_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, + launch_args: t.Optional[StringArgument] = None, env_vars: t.Optional[StringArgument] = None, ) -> None: try: self._launcher = LauncherType(launcher) - except KeyError: + except ValueError: raise ValueError(f"Invalid launcher type: {launcher}") - self._arg_translator = self._get_launcher() - - if env_vars: - validate_env_vars(env_vars) + self._arg_translator = self._get_arg_builder(launch_args) self.env_vars = env_vars or {} - - if launcher_args: - validate_args(launcher_args) - self.launcher_args = launcher_args or {} @property - def launcher(self): - return self._launcher - - @property - def arg_translator(self): - return self._arg_translator - + def launcher(self) -> str: + """Return the launcher name. + """ + return self._launcher.value + @property - def launcher_args(self) -> t.Dict[str, t.Union[int, str, float, None]]: - """Return an immutable list of attached launcher arguments. - - :returns: attached run arguments + def launch_args(self) -> LaunchArgTranslator: + """Return the launch argument translator. """ - return self._launcher_args - - @launcher_args.setter - def launcher_args(self, value: t.Dict[str, t.Union[int, str, float,None]]) -> None: - """Set the launcher arguments. + # Is a deep copy needed here? + return self._arg_translator - :param value: run arguments + @launch_args.setter + def launch_args(self, args: t.Mapping[str, str]) -> None: + """Update the launch arguments. """ - self._launcher_args = copy.deepcopy(value) + self.launch_args._launch_args.clear() + for k, v in args.items(): + self.launch_args.set(k, v) @property def env_vars(self) -> StringArgument: """Return an immutable list of attached environment variables. - - :returns: attached environment variables """ - return self._env_vars + return copy.deepcopy(self._env_vars) @env_vars.setter - def env_vars(self, value: StringArgument) -> None: + def env_vars(self, value: t.Mapping[str, str]) -> None: """Set the environment variables. - - :param value: environment variables """ self._env_vars = copy.deepcopy(value) - @property - def reserved_launch_args(self): - return self.arg_translator.set_reserved_launch_args() - - def _get_launcher(self) -> LaunchArgTranslator: + def _get_arg_builder(self, launch_args) -> LaunchArgTranslator: """ Map the Launcher to the LaunchArgTranslator """ - if self._launcher.value == 'slurm': - return SlurmArgTranslator() - elif self._launcher.value == 'mpiexec': - return MpiexecArgTranslator() - elif self._launcher.value == 'mpirun': - return MpiArgTranslator() - elif self._launcher.value == 'orterun': - return OrteArgTranslator() - elif self._launcher.value == 'alps': - return AprunArgTranslator() - elif self._launcher.value == 'lsf': - return JsrunArgTranslator() - elif self._launcher.value == 'pals': - return PalsMpiexecArgTranslator() - elif self._launcher.value == 'dragon': - return DragonArgTranslator() - elif self._launcher.value == 'local': - return LocalArgTranslator() - - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return self.arg_translator.launcher_str() - - def set_nodes(self, nodes: int) -> None: - """ Sets the number of nodes - - :param nodes: The number of nodes - """ - - args = self.arg_translator.set_nodes(nodes) - - if args: - self.update_launcher_args(args) - - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - """ Specify the hostlist for this job - - :param host_list: hosts to launch on - """ - - args = self.arg_translator.set_hostlist(host_list) - - if args: - self.update_launcher_args(args) - - def set_hostlist_from_file(self, file_path: str) -> None: - """ Use the contents of a file to set the node list + if self._launcher == LauncherType.SlurmLauncher: + return SlurmArgTranslator(launch_args) + elif self._launcher == LauncherType.MpiexecLauncher: + return MpiexecArgTranslator(launch_args) + elif self._launcher == LauncherType.MpirunLauncher: + return MpiArgTranslator(launch_args) + elif self._launcher == LauncherType.OrterunLauncher: + return OrteArgTranslator(launch_args) + elif self._launcher == LauncherType.AlpsLauncher: + return AprunArgTranslator(launch_args) + elif self._launcher == LauncherType.LsfLauncher: + return JsrunArgTranslator(launch_args) + elif self._launcher == LauncherType.PalsLauncher: + return PalsMpiexecArgTranslator(launch_args) + elif self._launcher == LauncherType.DragonLauncher: + return DragonArgTranslator(launch_args) + elif self._launcher == LauncherType.LocalLauncher: + return LocalArgTranslator(launch_args) + else: + raise ValueError(f"Invalid launcher type: {self._launcher}") + + def update_env(self, env_vars: StringArgument) -> None: + """Update the job environment variables - :param file_path: Path to the hostlist file - """ - - args = self.arg_translator.set_hostlist_from_file(file_path) - - if args: - self.update_launcher_args(args) + To fully inherit the current user environment, add the + workload-manager-specific flag to the launch command through the + :meth:`add_exe_args` method. For example, ``--export=ALL`` for + slurm, or ``-V`` for PBS/aprun. - def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: - """ Specify a list of hosts to exclude for launching this job - :param host_list: hosts to exclude + :param env_vars: environment variables to update or add + :raises TypeError: if env_vars values cannot be coerced to strings """ - - args = self.arg_translator.set_excluded_hosts(host_list) - - if args: - self.update_launcher_args(args) + # Coerce env_vars values to str as a convenience to user + for env, val in env_vars.items(): + if not (isinstance(val, str) and isinstance(env, str)): + raise TypeError( + f"env_vars[{env}] was of type {type(val)}, not str" + ) + self._env_vars.update(env_vars) - def set_cpus_per_task(self, cpus_per_task: int) -> None: - """ Set the number of cpus to use per task - - :param cpus_per_task: number of cpus to use per task - """ - - args = self.arg_translator.set_cpus_per_task(cpus_per_task) - - if args: - self.update_launcher_args(args) - - def set_tasks(self, tasks: int) -> None: - """ Set the number of tasks for this job - - :param tasks: number of tasks - """ - args = self.arg_translator.set_tasks(tasks) - if args: - self.update_launcher_args(args) - - def set_tasks_per_node(self, tasks_per_node: int) -> None: - """ Set the number of tasks per node for this job - - :param tasks_per_node: number of tasks per node - """ - args = self.arg_translator.set_tasks_per_node(tasks_per_node) - if args: - self.update_launcher_args(args) - - def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: - """ Bind by setting CPU masks on tasks - - :param bindings: List specifing the cores to which MPI processes are bound - """ - args = self.arg_translator.set_cpu_bindings(bindings) - if args: - self.update_launcher_args(args) - - def set_memory_per_node(self, memory_per_node: int) -> None: - """ Specify the real memory required per node - - :param memory_per_node: Amount of memory per node in megabytes - """ - args = self.arg_translator.set_memory_per_node(memory_per_node) - if args: - self.update_launcher_args(args) - - def set_executable_broadcast(self, dest_path: str) -> None: - """ Copy executable file to allocated compute nodes - - :param dest_path: Path to copy an executable file - """ - args = self.arg_translator.set_executable_broadcast(dest_path) - if args: - self.update_launcher_args(args) - - def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: - """Specify the node feature for this job - - :param feature_list: node feature to launch on - :raises TypeError: if not str or list of str - """ - args = self.arg_translator.set_node_feature(feature_list) - if args: - self.update_launcher_args(args) - - def set_walltime(self, walltime: str) -> None: - """Set the walltime of the job - - :param walltime: wall time - """ - args = self.arg_translator.set_walltime(walltime) - if args: - self.update_launcher_args(args) - - def set_binding(self, binding: str) -> None: - """Set binding - - This sets ``--bind`` - - :param binding: Binding, e.g. `packed:21` - """ - args = self.arg_translator.set_binding(binding) - if args: - self.update_launcher_args(args) - - def set_cpu_binding_type(self, bind_type: str) -> None: - """Specifies the cores to which MPI processes are bound - - This sets ``--bind-to`` for MPI compliant implementations - - :param bind_type: binding type - """ - args = self.arg_translator.set_cpu_binding_type(bind_type) - if args: - self.update_launcher_args(args) - - def set_task_map(self, task_mapping: str) -> None: - """Set ``mpirun`` task mapping - - this sets ``--map-by `` - - For examples, see the man page for ``mpirun`` - - :param task_mapping: task mapping - """ - args = self.arg_translator.set_task_map(task_mapping) - if args: - self.update_launcher_args(args) - - def set_het_group(self, het_group: t.Iterable[int]) -> None: - """Set the heterogeneous group for this job - - this sets `--het-group` - - :param het_group: list of heterogeneous groups - """ - args = self.arg_translator.set_het_group(het_group) - if args: - self.update_launcher_args(args) - - def set_verbose_launch(self, verbose: bool) -> None: - """Set the job to run in verbose mode - - This sets ``--verbose`` - - :param verbose: Whether the job should be run verbosely - """ - args = self.arg_translator.set_verbose_launch(verbose) - if args and verbose: - self.update_launcher_args(args) - if args and not verbose: - self.launcher_args.pop(next(iter(args))) - - def set_quiet_launch(self, quiet: bool) -> None: - """Set the job to run in quiet mode - - This sets ``--quiet`` - - :param quiet: Whether the job should be run quietly + def format_env_vars(self) -> t.Union[t.List[str],None]: + """Build bash compatible environment variable string for Slurm + :returns: the formatted string of environment variables """ - args = self.arg_translator.set_quiet_launch(quiet) - if args and quiet: - self.update_launcher_args(args) - if args and not quiet: - self.launcher_args.pop(next(iter(args))) + return self._arg_translator.format_env_vars(self.env_vars) def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: """Build environment variable string for Slurm - Slurm takes exports in comma separated lists the list starts with all as to not disturb the rest of the environment for more information on this, see the slurm documentation for srun - :returns: the formatted string of environment variables """ - return self.arg_translator.format_comma_sep_env_vars(self.env_vars) + return self._arg_translator.format_comma_sep_env_vars(self.env_vars) - def format_launcher_args(self) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str],None]: """Return formatted launch arguments - For ``RunSettings``, the run arguments are passed literally with no formatting. - :return: list run arguments for these settings """ - return self.arg_translator.format_launcher_args(self.launcher_args) - - def format_env_vars(self) -> t.Union[t.List[str],None]: - """Build bash compatible environment variable string for Slurm - - :returns: the formatted string of environment variables - """ - return self.arg_translator.format_env_vars(self.env_vars) - - def update_env(self, env_vars: t.Dict[str, t.Union[str, int, float, bool]]) -> None: - """Update the job environment variables - - To fully inherit the current user environment, add the - workload-manager-specific flag to the launch command through the - :meth:`add_exe_args` method. For example, ``--export=ALL`` for - slurm, or ``-V`` for PBS/aprun. - - - :param env_vars: environment variables to update or add - :raises TypeError: if env_vars values cannot be coerced to strings - """ - val_types = (str, int, float, bool) - # Coerce env_vars values to str as a convenience to user - for env, val in env_vars.items(): - if not isinstance(val, val_types): - raise TypeError( - f"env_vars[{env}] was of type {type(val)}, not {val_types}" - ) - - self.env_vars[env] = str(val) - - def update_launcher_args(self, args: t.Mapping[str, int | str | float | None]) -> None: - self.launcher_args.update(args) - - def set(self, key: str, arg: t.Union[str,int,float,None]) -> None: - # Store custom arguments in the launcher_args - if not isinstance(key, str): - raise TypeError("Argument name should be of type str") - key = key.strip().lstrip("-") - if key in self.reserved_launch_args: - logger.warning( - ( - f"Could not set argument '{key}': " - f"it is a reserved argument of '{type(self).__name__}'" - ) - ) - return - if key in self.launcher_args and arg != self.launcher_args[key]: - logger.warning(f"Overwritting argument '{key}' with value '{arg}'") - self.launcher_args[key] = arg + return self._arg_translator.format_launch_args() def __str__(self) -> str: # pragma: no-cover - string = f"\nLauncher: {self.arg_translator.launcher_str}" - if self.launcher_args: - string += f"\nLaunch Arguments:\n{fmt_dict(self.launcher_args)}" + string = f"\nLauncher: {self.launcher}" + if self.launch_args._launch_args: + string += f"\nLaunch Arguments:\n{fmt_dict(self.launch_args._launch_args)}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" return string \ No newline at end of file diff --git a/smartsim/settings/translators/batch/lsf.py b/smartsim/settings/translators/batch/lsf.py index a721217a72..7313855520 100644 --- a/smartsim/settings/translators/batch/lsf.py +++ b/smartsim/settings/translators/batch/lsf.py @@ -36,12 +36,13 @@ class BsubBatchArgTranslator(BatchArgTranslator): - def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ - return SchedulerType.LsfScheduler.value + def __init__( + self, + scheduler_args: StringArgument, + ) -> None: + super().__init__(scheduler_args) - def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + def set_walltime(self, walltime: str) -> None: """Set the walltime This sets ``-W``. @@ -54,9 +55,9 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: if walltime: if len(walltime.split(":")) > 2: walltime = ":".join(walltime.split(":")[:2]) - return {"W": walltime} + self.set("W", walltime) - def set_smts(self, smts: int) -> t.Union[IntegerArgument,None]: + def set_smts(self, smts: int) -> None: """Set SMTs This sets ``-alloc_flags``. If the user sets @@ -65,18 +66,18 @@ def set_smts(self, smts: int) -> t.Union[IntegerArgument,None]: :param smts: SMT (e.g on Summit: 1, 2, or 4) """ - return {"alloc_flags": int(smts)} + self.set("alloc_flags", str(smts)) - def set_project(self, project: str) -> t.Union[StringArgument,None]: + def set_project(self, project: str) -> None: """Set the project This sets ``-P``. :param time: project name """ - return {"P": project} + self.set("P", project) - def set_account(self, account: str) -> t.Union[StringArgument,None]: + def set_account(self, account: str) -> None: """Set the project this function is an alias for `set_project`. @@ -85,16 +86,16 @@ def set_account(self, account: str) -> t.Union[StringArgument,None]: """ return self.set_project(account) - def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job This sets ``-nnodes``. :param nodes: number of nodes """ - return {"nnodes": int(num_nodes)} + self.set("nnodes", str(num_nodes)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job :param host_list: hosts to launch on @@ -106,34 +107,34 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"m": '"' + " ".join(host_list) + '"'} + self.set("m", '"' + " ".join(host_list) + '"') - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + def set_tasks(self, tasks: int) -> None: """Set the number of tasks for this job This sets ``-n`` :param tasks: number of tasks """ - return {"n": int(tasks)} + self.set("n", str(tasks)) - def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + def set_queue(self, queue: str) -> None: """Set the queue for this job This sets ``-q`` :param queue: The queue to submit the job on """ - return {"q": queue} + self.set("q", queue) - def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview :return: list of batch arguments for Qsub """ opts = [] - for opt, value in batch_args.items(): + for opt, value in self._scheduler_args.items(): prefix = "-" # LSF only uses single dashses @@ -142,4 +143,8 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] else: opts += [f"{prefix}{opt}", str(value)] - return opts \ No newline at end of file + return opts + + def set(self, key: str, value: str | None) -> None: + # Store custom arguments in the launcher_args + self._scheduler_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/batch/pbs.py b/smartsim/settings/translators/batch/pbs.py index 725376ee13..05d5b6d86b 100644 --- a/smartsim/settings/translators/batch/pbs.py +++ b/smartsim/settings/translators/batch/pbs.py @@ -37,12 +37,13 @@ class QsubBatchArgTranslator(BatchArgTranslator): - def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ - return SchedulerType.PbsScheduler.value + def __init__( + self, + scheduler_args: StringArgument, + ) -> None: + super().__init__(scheduler_args) - def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job In PBS, 'select' is the more primitive way of describing how @@ -54,9 +55,9 @@ def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: :param num_nodes: number of nodes """ - return {"nodes": num_nodes} + self.set("nodes", str(num_nodes)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job :param host_list: hosts to launch on @@ -68,9 +69,9 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be a list of strings") - return {"hostname": ",".join(host_list)} + self.set("hostname", ",".join(host_list)) - def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job format = "HH:MM:SS" @@ -81,16 +82,16 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: :param walltime: wall time """ - return {"walltime": walltime} + self.set("walltime", walltime) - def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + def set_queue(self, queue: str) -> None: """Set the queue for the batch job :param queue: queue name """ - return {"q": str(queue)} + self.set("q", str(queue)) - def set_ncpus(self, num_cpus: int) -> t.Union[IntegerArgument,None]: + def set_ncpus(self, num_cpus: int) -> None: """Set the number of cpus obtained in each node. If a select argument is provided in @@ -99,22 +100,22 @@ def set_ncpus(self, num_cpus: int) -> t.Union[IntegerArgument,None]: :param num_cpus: number of cpus per node in select """ - return {"ppn": int(num_cpus)} + self.set("ppn", str(num_cpus)) - def set_account(self, account: str) -> t.Union[StringArgument,None]: + def set_account(self, account: str) -> None: """Set the account for this batch job :param acct: account id """ - return {"A": str(account)} + self.set("A", str(account)) - def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview :return: batch arguments for Qsub :raises ValueError: if options are supplied without values """ - opts, batch_arg_copy = self._create_resource_list(batch_args) + opts, batch_arg_copy = self._create_resource_list(self._scheduler_args) for opt, value in batch_arg_copy.items(): prefix = "-" if not value: @@ -122,8 +123,9 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] opts += [f"{prefix}{opt}", str(value)] return opts + @staticmethod def _sanity_check_resources( - self, batch_args: t.Dict[str, t.Union[str,int,float,None]] + batch_args: t.Dict[str, t.Union[str,int,float,None]] ) -> None: """Check that only select or nodes was specified in resources @@ -141,10 +143,11 @@ def _sanity_check_resources( "'select' was set using 'set_resource'. Please only specify one." ) - if has_select and not isinstance(has_select, int): - raise TypeError("The value for 'select' must be an integer") - if has_nodes and not isinstance(has_nodes, int): - raise TypeError("The value for 'nodes' must be an integer") + # TODO ask the team if this is true below + # if has_select and not isinstance(has_select, int): + # raise TypeError("The value for 'select' must be an integer") + # if has_nodes and not isinstance(has_nodes, int): + # raise TypeError("The value for 'nodes' must be an integer") for key, value in batch_args.items(): if not isinstance(key, str): @@ -181,8 +184,8 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No if walltime := batch_arg_copy.pop("walltime", None): res += ["-l", f"walltime={walltime}"] - # # All other "standard" resource specs - # for resource, value in batch_arg_copy.items(): - # res += [f"-l {resource}={value}"] + return res, batch_arg_copy - return res, batch_arg_copy \ No newline at end of file + def set(self, key: str, value: str | None) -> None: + # Store custom arguments in the launcher_args + self._scheduler_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/batch/slurm.py b/smartsim/settings/translators/batch/slurm.py index 03e704bdad..2d33e4c76a 100644 --- a/smartsim/settings/translators/batch/slurm.py +++ b/smartsim/settings/translators/batch/slurm.py @@ -29,20 +29,20 @@ import re import typing as t from ..batchArgTranslator import BatchArgTranslator -from ...common import IntegerArgument, StringArgument -from ...batchCommand import SchedulerType +from ...common import StringArgument from smartsim.log import get_logger logger = get_logger(__name__) class SlurmBatchArgTranslator(BatchArgTranslator): - def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ - return SchedulerType.SlurmScheduler.value + def __init__( + self, + scheduler_args: StringArgument, + ) -> None: + super().__init__(scheduler_args) - def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job format = "HH:MM:SS" @@ -51,38 +51,38 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: """ pattern = r'^\d{2}:\d{2}:\d{2}$' if walltime and re.match(pattern, walltime): - return {"time": str(walltime)} + self.set("time", str(walltime)) else: raise ValueError("Invalid walltime format. Please use 'HH:MM:SS' format.") - def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job This sets ``--nodes``. :param num_nodes: number of nodes """ - return {"nodes": int(num_nodes)} + self.set("nodes", str(num_nodes)) - def set_account(self, account: str) -> t.Union[StringArgument,None]: + def set_account(self, account: str) -> None: """Set the account for this batch job This sets ``--account``. :param account: account id """ - return {"account": account} + self.set("account", account) - def set_partition(self, partition: str) -> t.Union[StringArgument,None]: + def set_partition(self, partition: str) -> None: """Set the partition for the batch job This sets ``--partition``. :param partition: partition name """ - return {"partition": str(partition)} + self.set("partition", str(partition)) - def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + def set_queue(self, queue: str) -> None: """alias for set_partition Sets the partition for the slurm batch job @@ -91,16 +91,16 @@ def set_queue(self, queue: str) -> t.Union[StringArgument,None]: """ return self.set_partition(queue) - def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: + def set_cpus_per_task(self, cpus_per_task: int) -> None: """Set the number of cpus to use per task This sets ``--cpus-per-task`` :param num_cpus: number of cpus to use per task """ - return {"cpus-per-task": int(cpus_per_task)} + self.set("cpus-per-task", str(cpus_per_task)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job This sets ``--nodelist``. @@ -114,16 +114,16 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"nodelist": ",".join(host_list)} + self.set("nodelist", ",".join(host_list)) - def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview :return: batch arguments for Sbatch """ opts = [] # TODO add restricted here - for opt, value in batch_args.items(): + for opt, value in self._scheduler_args.items(): # attach "-" prefix if argument is 1 character otherwise "--" short_arg = len(opt) == 1 prefix = "-" if short_arg else "--" @@ -135,4 +135,8 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] opts += [prefix + opt, str(value)] else: opts += ["=".join((prefix + opt, str(value)))] - return opts \ No newline at end of file + return opts + + def set(self, key: str, value: str | None) -> None: + # Store custom arguments in the launcher_args + self._scheduler_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/batchArgTranslator.py b/smartsim/settings/translators/batchArgTranslator.py index a34ef93399..8307d22121 100644 --- a/smartsim/settings/translators/batchArgTranslator.py +++ b/smartsim/settings/translators/batchArgTranslator.py @@ -29,6 +29,7 @@ from abc import ABC, abstractmethod import typing as t from ..common import IntegerArgument, StringArgument +import copy from smartsim.log import get_logger @@ -40,31 +41,20 @@ class BatchArgTranslator(ABC): responsibility of child classes for each launcher to translate the input parameter to a properly formatted launcher argument. """ + + def __init__(self, scheduler_args) -> None: + self._scheduler_args = copy.deepcopy(scheduler_args) or {} @abstractmethod - def scheduler_str(self) -> str: - """ Get the string representation of the launcher - """ - pass - - @abstractmethod - def set_account(self, account: str) -> t.Union[StringArgument,None]: + def set_account(self, account: str) -> None: """Set the account for this batch job :param account: account id """ pass - def set_partition(self, partition: str) -> t.Union[StringArgument,None]: - """Set the partition for the batch job - - :param partition: partition name - """ - logger.warning(f"set_partition() not supported for {self.scheduler_str()}.") - return None - @abstractmethod - def set_queue(self, queue: str) -> t.Union[StringArgument,None]: + def set_queue(self, queue: str) -> None: """alias for set_partition Sets the partition for the slurm batch job @@ -73,30 +63,8 @@ def set_queue(self, queue: str) -> t.Union[StringArgument,None]: """ pass - def set_smts(self, smts: int) -> t.Union[IntegerArgument,None]: - """Set SMTs - - This sets ``-alloc_flags``. If the user sets - SMT explicitly through ``-alloc_flags``, then that - takes precedence. - - :param smts: SMT (e.g on Summit: 1, 2, or 4) - """ - logger.warning(f"set_smts() not supported for {self.scheduler_str()}.") - return None - - def set_project(self, project: str) -> t.Union[StringArgument,None]: - """Set the project - - This sets ``-P``. - - :param time: project name - """ - logger.warning(f"set_project() not supported for {self.scheduler_str()}.") - return None - @abstractmethod - def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job :param walltime: wall time @@ -104,23 +72,15 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: pass @abstractmethod - def set_nodes(self, num_nodes: int) -> t.Union[IntegerArgument,None]: + def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job :param num_nodes: number of nodes """ pass - def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: - """Set the number of cpus to use per task - - :param num_cpus: number of cpus to use per task - """ - logger.warning(f"set_cpus_per_task() not supported for {self.scheduler_str()}.") - return None - @abstractmethod - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job :param host_list: hosts to launch on @@ -134,24 +94,4 @@ def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]] :return: batch arguments for Sbatch """ - pass - - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: - """Set the number of tasks for this job - - :param tasks: number of tasks - """ - logger.warning(f"set_tasks() not supported for {self.scheduler_str()}.") - return None - - def set_ncpus(self, num_cpus: int) -> t.Union[IntegerArgument,None]: - """Set the number of cpus obtained in each node. - - If a select argument is provided in - ``QsubBatchSettings.resources``, then - this value will be overridden - - :param num_cpus: number of cpus per node in select - """ - logger.warning(f"set_ncpus() not supported for {self.scheduler_str()}.") - return None \ No newline at end of file + pass \ No newline at end of file diff --git a/smartsim/settings/translators/launch/__init__.py b/smartsim/settings/translators/launch/__init__.py index 8deaa227cf..3eba5e05db 100644 --- a/smartsim/settings/translators/launch/__init__.py +++ b/smartsim/settings/translators/launch/__init__.py @@ -6,14 +6,14 @@ from .pals import PalsMpiexecArgTranslator from .slurm import SlurmArgTranslator -# __all__ = [ -# "AprunArgTranslator", -# "DragonArgTranslator", -# "LocalArgTranslator", -# "JsrunArgTranslator", -# "MpiArgTranslator", -# "MpiexecArgTranslator", -# "OrteArgTranslator", -# "PalsMpiexecArgTranslator", -# "SlurmArgTranslator", -# ] \ No newline at end of file +__all__ = [ + "AprunArgTranslator", + "DragonArgTranslator", + "LocalArgTranslator", + "JsrunArgTranslator", + "MpiArgTranslator", + "MpiexecArgTranslator", + "OrteArgTranslator", + "PalsMpiexecArgTranslator", + "SlurmArgTranslator", +] \ No newline at end of file diff --git a/smartsim/settings/translators/launch/alps.py b/smartsim/settings/translators/launch/alps.py index 7b8944ecbf..9ad35c58d0 100644 --- a/smartsim/settings/translators/launch/alps.py +++ b/smartsim/settings/translators/launch/alps.py @@ -27,52 +27,52 @@ from __future__ import annotations from ..launchArgTranslator import LaunchArgTranslator import typing as t -from ...common import IntegerArgument, StringArgument -from ...launchCommand import LauncherType +from ...common import set_check_input, StringArgument from smartsim.log import get_logger logger = get_logger(__name__) class AprunArgTranslator(LaunchArgTranslator): - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.AlpsLauncher.value + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) - def set_reserved_launch_args(self) -> set[str]: + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ - return set() + return {"wdir"} - def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: + def set_cpus_per_task(self, cpus_per_task: int) -> None: """Set the number of cpus to use per task This sets ``--cpus-per-pe`` :param cpus_per_task: number of cpus to use per task """ - return {"cpus-per-pe": int(cpus_per_task)} + self.set("cpus-per-pe",str(cpus_per_task)) - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + def set_tasks(self, tasks: int) -> None: """Set the number of tasks for this job This sets ``--pes`` :param tasks: number of tasks """ - return {"pes": int(tasks)} + self.set("pes",str(tasks)) - def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + def set_tasks_per_node(self, tasks_per_node: int) -> None: """Set the number of tasks for this job This sets ``--pes-per-node`` :param tasks_per_node: number of tasks per node """ - return {"pes-per-node": int(tasks_per_node)} + self.set("pes-per-node",str(tasks_per_node)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job This sets ``--node-list`` @@ -86,18 +86,18 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"node-list": ",".join(host_list)} + self.set("node-list", ",".join(host_list)) - def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: + def set_hostlist_from_file(self, file_path: str) -> None: """Use the contents of a file to set the node list This sets ``--node-list-file`` :param file_path: Path to the hostlist file """ - return {"node-list-file": file_path} + self.set("node-list-file",file_path) - def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify a list of hosts to exclude for launching this job This sets ``--exclude-node-list`` @@ -111,9 +111,9 @@ def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[St raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"exclude-node-list": ",".join(host_list)} + self.set("exclude-node-list", ",".join(host_list)) - def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> t.Union[StringArgument, None]: + def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: """Specifies the cores to which MPI processes are bound This sets ``--cpu-binding`` @@ -122,34 +122,37 @@ def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> t.Union[Strin """ if isinstance(bindings, int): bindings = [bindings] - return {"cpu-binding": ",".join(str(int(num)) for num in bindings)} + self.set("cpu-binding", ",".join(str(num) for num in bindings)) - def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument, None]: + def set_memory_per_node(self, memory_per_node: int) -> None: """Specify the real memory required per node This sets ``--memory-per-pe`` in megabytes :param memory_per_node: Per PE memory limit in megabytes """ - return {"memory-per-pe": str(memory_per_node)} + self.set("memory-per-pe", str(memory_per_node)) - def set_walltime(self, walltime: str) -> t.Union[StringArgument, None]: + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job Walltime is given in total number of seconds :param walltime: wall time """ - return {"cpu-time-limit": str(walltime)} + self.set("cpu-time-limit", str(walltime)) - def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + def set_verbose_launch(self, verbose: bool) -> None: """Set the job to run in verbose mode This sets ``--debug`` arg to the highest level :param verbose: Whether the job should be run verbosely """ - return {"debug": 7} + if verbose: + self.set("debug","7") + else: + self._launch_args.pop("debug", None) def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None],None]: """Set the job to run in quiet mode @@ -158,7 +161,10 @@ def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None],None]: :param quiet: Whether the job should be run quietly """ - return {"quiet": None} + if quiet: + self.set("quiet",None) + else: + self._launch_args.pop("quiet", None) def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: """Format the environment variables for aprun @@ -171,24 +177,38 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> formatted += ["-e", name + "=" + str(value)] return formatted - def format_launcher_args(self, launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str],None]: """Return a list of ALPS formatted run arguments :return: list of ALPS arguments for these settings """ # args launcher uses args = [] - restricted = ["wdir"] - - for opt, value in launch_args.items(): - if opt not in restricted: - short_arg = bool(len(str(opt)) == 1) - prefix = "-" if short_arg else "--" - if not value: - args += [prefix + opt] + + for opt, value in self._launch_args.items(): + short_arg = bool(len(str(opt)) == 1) + prefix = "-" if short_arg else "--" + if not value: + args += [prefix + opt] + else: + if short_arg: + args += [prefix + opt, str(value)] else: - if short_arg: - args += [prefix + opt, str(value)] - else: - args += ["=".join((prefix + opt, str(value)))] - return args \ No newline at end of file + args += ["=".join((prefix + opt, str(value)))] + return args + + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments + """ + set_check_input(key,value,logger) + if key in self._reserved_launch_args(): + logger.warning( + ( + f"Could not set argument '{key}': " + f"it is a reserved argument of '{type(self).__name__}'" + ) + ) + return + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/dragon.py b/smartsim/settings/translators/launch/dragon.py index c22241d654..abd0f63b3f 100644 --- a/smartsim/settings/translators/launch/dragon.py +++ b/smartsim/settings/translators/launch/dragon.py @@ -28,34 +28,37 @@ import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument -from ...launchCommand import LauncherType +from ...common import StringArgument, set_check_input from smartsim.log import get_logger logger = get_logger(__name__) class DragonArgTranslator(LaunchArgTranslator): + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.DragonLauncher.value - - def set_reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ - return set() - - def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: + def set_nodes(self, nodes: int) -> None: """Set the number of nodes :param nodes: number of nodes to run with """ - return {"nodes": nodes} + self.set("nodes",str(nodes)) - def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + def set_tasks_per_node(self, tasks_per_node: int) -> None: """Set the number of tasks for this job :param tasks_per_node: number of tasks per node """ - return {"tasks-per-node": tasks_per_node} \ No newline at end of file + self.set("tasks-per-node",str(tasks_per_node)) + + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments + """ + set_check_input(key,value,logger) + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/local.py b/smartsim/settings/translators/launch/local.py index 48d21999eb..6f0ce16304 100644 --- a/smartsim/settings/translators/launch/local.py +++ b/smartsim/settings/translators/launch/local.py @@ -28,23 +28,18 @@ import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...launchCommand import LauncherType from smartsim.log import get_logger -from ...common import StringArgument +from ...common import StringArgument, set_check_input logger = get_logger(__name__) class LocalArgTranslator(LaunchArgTranslator): - - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.LocalLauncher.value - - def set_reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ - return set() + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: """Build environment variable string @@ -59,13 +54,21 @@ def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None] formatted.append(f"{key}={val}") return formatted - def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str],None]: """Build launcher argument string :returns: formatted list of launcher arguments """ formatted = [] - for arg, value in launcher_args.items(): + for arg, value in self._launch_args.items(): formatted.append(arg) formatted.append(str(value)) - return formatted \ No newline at end of file + return formatted + + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments + """ + set_check_input(key,value,logger) + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/lsf.py b/smartsim/settings/translators/launch/lsf.py index d0095232c6..e886868c68 100644 --- a/smartsim/settings/translators/launch/lsf.py +++ b/smartsim/settings/translators/launch/lsf.py @@ -28,41 +28,42 @@ import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument +from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from smartsim.log import get_logger logger = get_logger(__name__) class JsrunArgTranslator(LaunchArgTranslator): - - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.LsfLauncher.value - - def set_reserved_launch_args(self) -> set[str]: + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) + + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return {"chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"} - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: + def set_tasks(self, tasks: int) -> None: """Set the number of tasks for this job This sets ``--np`` :param tasks: number of tasks """ - return {"np": int(tasks)} + self.set("np", str(tasks)) - def set_binding(self, binding: str) -> t.Union[StringArgument, None]: + def set_binding(self, binding: str) -> None: """Set binding This sets ``--bind`` :param binding: Binding, e.g. `packed:21` """ - return {"bind": binding} + self.set("bind", binding) def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: """Format environment variables. Each variable needs @@ -79,24 +80,38 @@ def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.L format_str += ["-E", f"{k}"] return format_str - def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str],None]: """Return a list of LSF formatted run arguments :return: list of LSF arguments for these settings """ # args launcher uses args = [] - restricted = ["chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"] - for opt, value in launcher_args.items(): - if opt not in restricted: - short_arg = bool(len(str(opt)) == 1) - prefix = "-" if short_arg else "--" - if value is None: - args += [prefix + opt] + for opt, value in self._launch_args.items(): + short_arg = bool(len(str(opt)) == 1) + prefix = "-" if short_arg else "--" + if value is None: + args += [prefix + opt] + else: + if short_arg: + args += [prefix + opt, str(value)] else: - if short_arg: - args += [prefix + opt, str(value)] - else: - args += ["=".join((prefix + opt, str(value)))] - return args \ No newline at end of file + args += ["=".join((prefix + opt, str(value)))] + return args + + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments + """ + set_check_input(key,value,logger) + if key in self._reserved_launch_args(): + logger.warning( + ( + f"Could not set argument '{key}': " + f"it is a reserved argument of '{type(self).__name__}'" + ) + ) + return + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/mpi.py b/smartsim/settings/translators/launch/mpi.py index a69389c10e..8b56b075c5 100644 --- a/smartsim/settings/translators/launch/mpi.py +++ b/smartsim/settings/translators/launch/mpi.py @@ -28,20 +28,26 @@ import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument +from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from smartsim.log import get_logger logger = get_logger(__name__) class _BaseMPIArgTranslator(LaunchArgTranslator): + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) - def set_reserved_launch_args(self) -> set[str]: + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return {"wd", "wdir"} - def set_task_map(self, task_mapping: str) -> t.Union[StringArgument, None]: + def set_task_map(self, task_mapping: str) -> None: """ Set ``mpirun`` task mapping this sets ``--map-by `` @@ -50,9 +56,9 @@ def set_task_map(self, task_mapping: str) -> t.Union[StringArgument, None]: :param task_mapping: task mapping """ - return {"map-by": task_mapping} + self.set("map-by",task_mapping) - def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, None]: + def set_cpus_per_task(self, cpus_per_task: int) -> None: """ Set the number of tasks for this job This sets ``--cpus-per-proc`` for MPI compliant implementations @@ -62,9 +68,9 @@ def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument, Non :param cpus_per_task: number of tasks """ - return {"cpus-per-proc": int(cpus_per_task)} + self.set("cpus-per-proc", str(cpus_per_task)) - def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument, None]: + def set_executable_broadcast(self, dest_path: str) -> None: """Copy the specified executable(s) to remote machines This sets ``--preload-binary`` @@ -78,34 +84,34 @@ def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument, No "Using session directory instead" ) ) - return {"preload-binary": dest_path} + self.set("preload-binary",dest_path) - def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument, None]: + def set_cpu_binding_type(self, bind_type: str) -> None: """ Specifies the cores to which MPI processes are bound This sets ``--bind-to`` for MPI compliant implementations :param bind_type: binding type """ - return {"bind-to": bind_type} + self.set("bind-to",bind_type) - def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + def set_tasks_per_node(self, tasks_per_node: int) -> None: """ Set the number of tasks per node :param tasks_per_node: number of tasks to launch per node """ - return {"npernode": int(tasks_per_node)} + self.set("npernode",str(tasks_per_node)) - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: + def set_tasks(self, tasks: int) -> None: """ Set the number of tasks for this job This sets ``-n`` for MPI compliant implementations :param tasks: number of tasks """ - return {"n": int(tasks)} + self.set("n", str(tasks)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """ Set the hostlist for the ``mpirun`` command This sets ``--host`` @@ -119,43 +125,49 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringA raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"host": ",".join(host_list)} + self.set("host", ",".join(host_list)) - def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: + def set_hostlist_from_file(self, file_path: str) -> None: """ Use the contents of a file to set the hostlist This sets ``--hostfile`` :param file_path: Path to the hostlist file """ - return {"hostfile": file_path} + self.set("hostfile",file_path) - def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + def set_verbose_launch(self, verbose: bool) -> None: """ Set the job to run in verbose mode This sets ``--verbose`` :param verbose: Whether the job should be run verbosely """ - return {"verbose": None} + if verbose: + self.set("verbose",None) + else: + self._launch_args.pop("verbose", None) - def set_walltime(self, walltime: str) -> t.Union[StringArgument, None]: + def set_walltime(self, walltime: str) -> None: """Set the maximum number of seconds that a job will run This sets ``--timeout`` :param walltime: number like string of seconds that a job will run in secs """ - return {"timeout": walltime} + self.set("timeout",walltime) - def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None], None]: + def set_quiet_launch(self, quiet: bool) -> None: """ Set the job to run in quiet mode This sets ``--quiet`` :param quiet: Whether the job should be run quietly """ - return {"quiet": None} + if quiet: + self.set("quiet",None) + else: + self._launch_args.pop("quiet", None) def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: """ Format the environment variables for mpirun @@ -173,41 +185,58 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> formatted += [env_string, name] return formatted - def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def format_launch_args(self) -> t.List[str]: """Return a list of MPI-standard formatted run arguments :return: list of MPI-standard arguments for these settings """ # args launcher uses args = [] - restricted = ["wdir", "wd"] - for opt, value in launcher_args.items(): - if opt not in restricted: - prefix = "--" - if not value: - args += [prefix + opt] - else: - args += [prefix + opt, str(value)] + for opt, value in self._launch_args.items(): + prefix = "--" + if not value: + args += [prefix + opt] + else: + args += [prefix + opt, str(value)] return args -class MpiArgTranslator(_BaseMPIArgTranslator): - - def launcher_str(self) -> str: - """ Get the string representation of the launcher + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments """ - return LauncherType.MpirunLauncher.value + set_check_input(key,value,logger) + if key in self._reserved_launch_args(): + logger.warning( + ( + f"Could not set argument '{key}': " + f"it is a reserved argument of '{type(self).__name__}'" + ) + ) + return + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value -class MpiexecArgTranslator(_BaseMPIArgTranslator): +class MpiArgTranslator(_BaseMPIArgTranslator): + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.MpiexecLauncher.value +class MpiexecArgTranslator(_BaseMPIArgTranslator): + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) class OrteArgTranslator(_BaseMPIArgTranslator): - - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.OrterunLauncher.value \ No newline at end of file + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) \ No newline at end of file diff --git a/smartsim/settings/translators/launch/pals.py b/smartsim/settings/translators/launch/pals.py index 5527059514..a33eab6a81 100644 --- a/smartsim/settings/translators/launch/pals.py +++ b/smartsim/settings/translators/launch/pals.py @@ -28,59 +28,59 @@ import typing as t from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument -from ...launchCommand import LauncherType +from ...common import StringArgument, set_check_input from smartsim.log import get_logger logger = get_logger(__name__) class PalsMpiexecArgTranslator(LaunchArgTranslator): + + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.PalsLauncher.value - - def set_reserved_launch_args(self) -> set[str]: + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ - return set() + return {"wdir", "wd"} - def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: + def set_cpu_binding_type(self, bind_type: str) -> None: """ Specifies the cores to which MPI processes are bound This sets ``--bind-to`` for MPI compliant implementations :param bind_type: binding type """ - return {"bind-to": bind_type} + self.set("bind-to",bind_type) - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument, None]: + def set_tasks(self, tasks: int) -> None: """ Set the number of tasks :param tasks: number of total tasks to launch """ - return {"np": int(tasks)} + self.set("np",str(tasks)) - def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument, None]: + def set_executable_broadcast(self, dest_path: str) -> None: """Copy the specified executable(s) to remote machines This sets ``--transfer`` :param dest_path: Destination path (Ignored) """ - return {"transfer": str(dest_path)} + self.set("transfer", dest_path) - def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument, None]: + def set_tasks_per_node(self, tasks_per_node: int) -> None: """ Set the number of tasks per node This sets ``--ppn`` :param tasks_per_node: number of tasks to launch per node """ - return {"ppn": int(tasks_per_node)} + self.set("ppn",str(tasks_per_node)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument, None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """ Set the hostlist for the PALS ``mpiexec`` command This sets ``hosts`` @@ -94,26 +94,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"hosts": ",".join(host_list)} - - def format_launch_args(self, launcher_args: t.Dict[str, t.Union[str,int,float]]) -> t.Union[t.List[str],None]: - """ Return a list of MPI-standard formatted run arguments - - :return: list of MPI-standard arguments for these settings - """ - # args launcher uses - args = [] - restricted = ["wdir", "wd"] - - for opt, value in launcher_args.items(): - if opt not in restricted: - prefix = "--" - if not value: - args += [prefix + opt] - else: - args += [prefix + opt, str(value)] - - return args + self.set("hosts",",".join(host_list)) def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: """ Format the environment variables for mpirun @@ -135,21 +116,35 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> return formatted - def format_launcher_args(self, launch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def format_launch_args(self) -> t.List[str]: """Return a list of MPI-standard formatted launcher arguments :return: list of MPI-standard arguments for these settings """ # args launcher uses args = [] - restricted = ["wdir", "wd"] - for opt, value in launch_args.items(): - if opt not in restricted: - prefix = "--" - if not value: - args += [prefix + opt] - else: - args += [prefix + opt, str(value)] + for opt, value in self._launch_args.items(): + prefix = "--" + if not value: + args += [prefix + opt] + else: + args += [prefix + opt, str(value)] + + return args - return args \ No newline at end of file + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments + """ + set_check_input(key,value,logger) + if key in self._reserved_launch_args(): + logger.warning( + ( + f"Could not set argument '{key}': " + f"it is a reserved argument of '{type(self).__name__}'" + ) + ) + return + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/slurm.py b/smartsim/settings/translators/launch/slurm.py index 5297caf096..5de63b511c 100644 --- a/smartsim/settings/translators/launch/slurm.py +++ b/smartsim/settings/translators/launch/slurm.py @@ -30,26 +30,25 @@ import re import os from ..launchArgTranslator import LaunchArgTranslator -from ...common import IntegerArgument, StringArgument -from ...launchCommand import LauncherType +from ...common import IntegerArgument, StringArgument, set_check_input from smartsim.log import get_logger logger = get_logger(__name__) - class SlurmArgTranslator(LaunchArgTranslator): - - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.SlurmLauncher.value - def set_reserved_launch_args(self) -> set[str]: + def __init__( + self, + launch_args: StringArgument, + ) -> None: + super().__init__(launch_args) + + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ return {"chdir", "D"} - def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: + def set_nodes(self, nodes: int) -> None: """ Set the number of nodes Effectively this is setting: ``srun --nodes `` @@ -57,9 +56,9 @@ def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: :param nodes: nodes to launch on :return: launcher argument """ - return {"nodes": int(nodes)} + self.set("nodes", str(nodes)) - def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """ Specify the hostlist for this job This sets ``--nodelist`` @@ -73,18 +72,18 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringAr raise TypeError("host_list argument must be a string or list of strings") elif not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return {"nodelist": ",".join(host_list)} + self.set("nodelist", ",".join(host_list)) - def set_hostlist_from_file(self, file_path: str) -> t.Union[StringArgument, None]: + def set_hostlist_from_file(self, file_path: str) -> None: """ Use the contents of a file to set the node list This sets ``--nodefile`` :param file_path: Path to the nodelist file """ - return {"nodefile": file_path} + self.set("nodefile", file_path) - def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: """ Specify a list of hosts to exclude for launching this job :param host_list: hosts to exclude @@ -96,36 +95,36 @@ def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> t.Union[S raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - return { "exclude": ",".join(host_list)} + self.set("exclude",",".join(host_list)) - def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: + def set_cpus_per_task(self, cpus_per_task: int) -> None: """ Set the number of cpus to use per task This sets ``--cpus-per-task`` :param num_cpus: number of cpus to use per task """ - return {"cpus-per-task": int(cpus_per_task)} + self.set("cpus-per-task", str(cpus_per_task)) - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: + def set_tasks(self, tasks: int) -> None: """ Set the number of tasks for this job This sets ``--ntasks`` :param tasks: number of tasks """ - return {"ntasks": int(tasks)} + self.set("ntasks",str(tasks)) - def set_tasks_per_node(self, tasks_per_node: int) -> t.Union[IntegerArgument,None]: + def set_tasks_per_node(self, tasks_per_node: int) -> None: """ Set the number of tasks for this job This sets ``--ntasks-per-node`` :param tasks_per_node: number of tasks per node """ - return {"ntasks-per-node": int(tasks_per_node)} + self.set("ntasks-per-node",str(tasks_per_node)) - def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> t.Union[StringArgument,None]: + def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> None: """ Bind by setting CPU masks on tasks This sets ``--cpu-bind`` using the ``map_cpu:`` option @@ -134,27 +133,27 @@ def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> t.Union[String """ if isinstance(bindings, int): bindings = [bindings] - return {"cpu_bind": "map_cpu:" + ",".join(str(num) for num in bindings)} + self.set("cpu_bind","map_cpu:" + ",".join(str(num) for num in bindings)) - def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument,None]: + def set_memory_per_node(self, memory_per_node: int) -> None: """ Specify the real memory required per node This sets ``--mem`` in megabytes :param memory_per_node: Amount of memory per node in megabytes """ - return {"mem": f"{memory_per_node}M"} + self.set("mem",f"{memory_per_node}M") - def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument,None]: + def set_executable_broadcast(self, dest_path: str) -> None: """ Copy executable file to allocated compute nodes This sets ``--bcast`` :param dest_path: Path to copy an executable file """ - return {"bcast": dest_path} + self.set("bcast",dest_path) - def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: + def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: """ Specify the node feature for this job This sets ``-C`` @@ -166,9 +165,9 @@ def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> t.Union[S feature_list = [feature_list.strip()] elif not all(isinstance(feature, str) for feature in feature_list): raise TypeError("node_feature argument must be string or list of strings") - return {"C": ",".join(feature_list)} + self.set("C",",".join(feature_list)) - def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: + def set_walltime(self, walltime: str) -> None: """ Set the walltime of the job format = "HH:MM:SS" @@ -177,11 +176,11 @@ def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: """ pattern = r'^\d{2}:\d{2}:\d{2}$' if walltime and re.match(pattern, walltime): - return {"time": str(walltime)} + self.set("time",str(walltime)) else: raise ValueError("Invalid walltime format. Please use 'HH:MM:SS' format.") - def set_het_group(self, het_group: t.Iterable[int]) -> t.Union[StringArgument,None]: + def set_het_group(self, het_group: t.Iterable[int]) -> None: """Set the heterogeneous group for this job this sets `--het-group` @@ -199,44 +198,49 @@ def set_het_group(self, het_group: t.Iterable[int]) -> t.Union[StringArgument,No f"but max het group in allocation is {het_size-1}" ) raise ValueError(msg) - return {"het-group": ",".join(str(group) for group in het_group)} + self.set("het-group",",".join(str(group) for group in het_group)) - def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: + def set_verbose_launch(self, verbose: bool) -> None: """ Set the job to run in verbose mode This sets ``--verbose`` :param verbose: Whether the job should be run verbosely """ - return {"verbose": None} + if verbose: + self.set("verbose",None) + else: + self._launch_args.pop("verbose", None) - def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str, None], None]: + def set_quiet_launch(self, quiet: bool) -> None: """Set the job to run in quiet mode This sets ``--quiet`` :param quiet: Whether the job should be run quietly """ - return {"quiet": None} + if quiet: + self.set("quiet",None) + else: + self._launch_args.pop("quiet", None) - def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str],None]: """Return a list of slurm formatted launch arguments :return: list of slurm arguments for these settings """ - # add additional slurm arguments based on key length - opts = [] - for opt, value in launcher_args.items(): - short_arg = bool(len(str(opt)) == 1) + formatted = [] + for key, value in self._launch_args.items(): + short_arg = bool(len(str(key)) == 1) prefix = "-" if short_arg else "--" if not value: - opts += [prefix + opt] + formatted += [prefix + key] else: if short_arg: - opts += [prefix + opt, str(value)] + formatted += [prefix + key, str(value)] else: - opts += ["=".join((prefix + opt, str(value)))] - return opts + formatted += ["=".join((prefix + key, str(value)))] + return formatted def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: """Build bash compatible environment variable string for Slurm @@ -291,4 +295,20 @@ def _check_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> None: "consider removing the variable from the environment " "and re-run the experiment." ) - logger.warning(msg) \ No newline at end of file + logger.warning(msg) + + def set(self, key: str, value: str | None) -> None: + """ Set the launch arguments + """ + set_check_input(key,value,logger) + if key in self._reserved_launch_args(): + logger.warning( + ( + f"Could not set argument '{key}': " + f"it is a reserved argument of '{type(self).__name__}'" + ) + ) + return + if key in self._launch_args and key != self._launch_args[key]: + logger.warning(f"Overwritting argument '{key}' with value '{value}'") + self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launchArgTranslator.py b/smartsim/settings/translators/launchArgTranslator.py index d56ff7d621..fcc5e3379a 100644 --- a/smartsim/settings/translators/launchArgTranslator.py +++ b/smartsim/settings/translators/launchArgTranslator.py @@ -27,193 +27,23 @@ from __future__ import annotations from abc import ABC, abstractmethod -import typing as t +import copy from smartsim.log import get_logger logger = get_logger(__name__) -from ..common import IntegerArgument, StringArgument - - class LaunchArgTranslator(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate the input parameter to a properly formatted launcher argument. """ - - @abstractmethod - def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - pass + def __init__(self, launch_args) -> None: + self._launch_args = copy.deepcopy(launch_args) or {} @abstractmethod - def set_reserved_launch_args(self) -> set[str]: - """ Reserved launch keys per Launcher. - """ - pass - - def set_nodes(self, nodes: int) -> t.Union[IntegerArgument, None]: - """ Convert the provide number of nodes into a properly formatted launcher - argument. - """ - logger.warning(f"set_nodes() not supported for {self.launcher_str()}.") - return None - - def set_hostlist(self, hostlist: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: - """ Convert the provide hostlist into a properly formatted launcher argument. - """ - logger.warning(f"set_hostlist() not supported for {self.launcher_str()}.") - return None - - def set_hostlist_from_file(self, hostlist: str) -> t.Union[StringArgument,None]: - """ Convert the file path into a properly formatted launcher argument. - """ - logger.warning(f"set_hostlist_from_file() not supported for {self.launcher_str()}.") - return None - - def set_excluded_hosts(self, hostlist: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: - """ Convert the hostlist into a properly formatted launcher argument. - """ - logger.warning(f"set_excluded_hosts() not supported for {self.launcher_str()}.") - return None - - def set_cpus_per_task(self, cpus_per_task: int) -> t.Union[IntegerArgument,None]: - """ Convert the cpus_per_task into a properly formatted launcher argument. - """ - logger.warning(f"set_cpus_per_task() not supported for {self.launcher_str()}.") - return None - - def set_tasks(self, tasks: int) -> t.Union[IntegerArgument,None]: - """ Convert the tasks into a properly formatted launcher argument. - """ - logger.warning(f"set_tasks() not supported for {self.launcher_str()}.") - return None - - def set_tasks_per_node(self, tasks: int) -> t.Union[IntegerArgument,None]: - """ Convert the set_tasks_per_node into a properly formatted launcher argument. - """ - logger.warning(f"set_tasks_per_node() not supported for {self.launcher_str()}.") - return None - - def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> t.Union[StringArgument,None]: - """ Convert the cpu bindings into a properly formatted launcher argument. - """ - logger.warning(f"set_cpu_bindings() not supported for {self.launcher_str()}.") - return None - - def set_memory_per_node(self, memory_per_node: int) -> t.Union[StringArgument,None]: - """ Convert the real memory required per node into a properly formatted - launcher argument. - """ - logger.warning(f"set_memory_per_node() not supported for {self.launcher_str()}.") - return None - - def set_executable_broadcast(self, dest_path: str) -> t.Union[StringArgument,None]: - """ Convert executable file to be copied to allocated compute nodes into - a properly formatted launcher argument. - """ - logger.warning(f"set_executable_broadcast() not supported for {self.launcher_str()}.") - return None - - def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> t.Union[StringArgument,None]: - """ Convert node feature into a properly formatted launcher argument. - """ - logger.warning(f"set_node_feature() not supported for {self.launcher_str()}.") - return None - - def set_walltime(self, walltime: str) -> t.Union[StringArgument,None]: - """ Convert walltime into a properly formatted launcher argument. - """ - logger.warning(f"set_walltime() not supported for {self.launcher_str()}.") - return None - - def set_binding(self, binding: str) -> t.Union[StringArgument,None]: - """Set binding - - This sets ``--bind`` - - :param binding: Binding, e.g. `packed:21` - """ - logger.warning(f"set_binding() not supported for {self.launcher_str()}.") - return None - - def set_cpu_binding_type(self, bind_type: str) -> t.Union[StringArgument,None]: - """Specifies the cores to which MPI processes are bound - - This sets ``--bind-to`` for MPI compliant implementations - - :param bind_type: binding type - """ - logger.warning(f"set_cpu_binding_type() not supported for {self.launcher_str()}.") - return None - - def set_task_map(self, task_mapping: str) -> t.Union[StringArgument,None]: - """Set ``mpirun`` task mapping - - this sets ``--map-by `` - - For examples, see the man page for ``mpirun`` - - :param task_mapping: task mapping - """ - logger.warning(f"set_task_map() not supported for {self.launcher_str()}.") - return None - - def set_het_group(self, het_group: t.Iterable[int]) -> t.Union[StringArgument, None]: - """Set the heterogeneous group for this job - - this sets `--het-group` - - :param het_group: list of heterogeneous groups - """ - logger.warning(f"set_het_group() not supported for {self.launcher_str()}.") - return None - - def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str, None], None]: - """Set the job to run in quiet mode - - This sets ``--quiet`` - - :param quiet: Whether the job should be run quietly - """ - logger.warning(f"set_quiet_launch() not supported for {self.launcher_str()}.") - return None - - def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: - """Build bash compatible environment variable string for Slurm - - :returns: the formatted string of environment variables - """ - logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") - return None - - def format_launcher_args(self, launcher_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Union[t.List[str],None]: - """ Build formatted launch arguments - """ - logger.warning(f"format_launcher_args() not supported for {self.launcher_str()}.") - return None - - def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.Tuple[str, t.List[str]],None]: - """Build environment variable string for Slurm - - Slurm takes exports in comma separated lists - the list starts with all as to not disturb the rest of the environment - for more information on this, see the slurm documentation for srun - - :returns: the formatted string of environment variables - """ - logger.warning(f"format_comma_sep_env_vars() not supported for {self.launcher_str()}.") - return None - - def set_verbose_launch(self, verbose: bool) -> t.Union[t.Dict[str, None], t.Dict[str, int], None]: - """Set the job to run in verbose mode - - This sets ``--verbose`` - - :param verbose: Whether the job should be run verbosely + def set(self, arg: str, val: str | None) -> None: + """ Set the launch arguments """ - logger.warning(f"set_verbose_launch() not supported for {self.launcher_str()}.") - return None \ No newline at end of file + pass \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 2f83311f2d..c47571b231 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -2,24 +2,13 @@ from smartsim.settings import LaunchSettings from smartsim.settings.translators.launch.alps import AprunArgTranslator from smartsim.settings.launchCommand import LauncherType -import logging - -def test_launcher_str(): - """Ensure launcher_str returns appropriate value""" - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert alpsLauncher.launcher_str() == LauncherType.AlpsLauncher.value - -def test_set_reserved_launcher_args(): - """Ensure launcher_str returns appropriate value""" - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert alpsLauncher.reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_cpus_per_task", (4,),4,"cpus-per-pe",id="set_cpus_per_task"), - pytest.param("set_tasks", (4,),4,"pes",id="set_tasks"), - pytest.param("set_tasks_per_node", (4,),4,"pes-per-node",id="set_tasks_per_node"), + pytest.param("set_cpus_per_task", (4,),"4","cpus-per-pe",id="set_cpus_per_task"), + pytest.param("set_tasks", (4,),"4","pes",id="set_tasks"), + pytest.param("set_tasks_per_node", (4,),"4","pes-per-node",id="set_tasks_per_node"), pytest.param("set_hostlist", ("host_A",),"host_A","node-list",id="set_hostlist_str"), pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","node-list",id="set_hostlist_list[str]"), pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","node-list-file",id="set_hostlist_from_file"), @@ -29,36 +18,36 @@ def test_set_reserved_launcher_args(): pytest.param("set_cpu_bindings", ([4,4],),"4,4","cpu-binding",id="set_cpu_bindings_list[str]"), pytest.param("set_memory_per_node", (8000,),"8000","memory-per-pe",id="set_memory_per_node"), pytest.param("set_walltime", ("10:00:00",),"10:00:00","cpu-time-limit",id="set_walltime"), - pytest.param("set_verbose_launch", (True,),7,"debug",id="set_verbose_launch"), + pytest.param("set_verbose_launch", (True,),"7","debug",id="set_verbose_launch"), pytest.param("set_quiet_launch", (True,),None,"quiet",id="set_quiet_launch"), ], ) def test_alps_class_methods(function, value, flag, result): alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) - getattr(alpsLauncher, function)(*value) - assert alpsLauncher.launcher_args[flag] == result + assert isinstance(alpsLauncher._arg_translator,AprunArgTranslator) + getattr(alpsLauncher.launch_args, function)(*value) + assert alpsLauncher.launch_args._launch_args[flag] == result def test_set_verbose_launch(): alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert isinstance(alpsLauncher.arg_translator,AprunArgTranslator) - alpsLauncher.set_verbose_launch(True) - assert alpsLauncher.launcher_args == {'debug': 7} - alpsLauncher.set_verbose_launch(False) - assert alpsLauncher.launcher_args == {} + assert isinstance(alpsLauncher._arg_translator,AprunArgTranslator) + alpsLauncher.launch_args.set_verbose_launch(True) + assert alpsLauncher.launch_args._launch_args == {'debug': "7"} + alpsLauncher.launch_args.set_verbose_launch(False) + assert alpsLauncher.launch_args._launch_args == {} def test_set_quiet_launch(): aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) - aprunLauncher.set_quiet_launch(True) - assert aprunLauncher.launcher_args == {'quiet': None} - aprunLauncher.set_quiet_launch(False) - assert aprunLauncher.launcher_args == {} + assert isinstance(aprunLauncher._arg_translator,AprunArgTranslator) + aprunLauncher.launch_args.set_quiet_launch(True) + assert aprunLauncher.launch_args._launch_args == {'quiet': None} + aprunLauncher.launch_args.set_quiet_launch(False) + assert aprunLauncher.launch_args._launch_args == {} def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher, env_vars=env_vars) - assert isinstance(aprunLauncher.arg_translator,AprunArgTranslator) + assert isinstance(aprunLauncher._arg_translator,AprunArgTranslator) aprunLauncher.update_env({"OMP_NUM_THREADS": "10"}) formatted = aprunLauncher.format_env_vars() result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] @@ -66,10 +55,10 @@ def test_format_env_vars(): def test_aprun_settings(): aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - aprunLauncher.set_cpus_per_task(2) - aprunLauncher.set_tasks(100) - aprunLauncher.set_tasks_per_node(20) - formatted = aprunLauncher.format_launcher_args() + aprunLauncher.launch_args.set_cpus_per_task(2) + aprunLauncher.launch_args.set_tasks(100) + aprunLauncher.launch_args.set_tasks_per_node(20) + formatted = aprunLauncher.format_launch_args() result = ["--cpus-per-pe=2", "--pes=100", "--pes-per-node=20"] assert formatted == result @@ -77,60 +66,18 @@ def test_invalid_hostlist_format(): """Test invalid hostlist formats""" alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) with pytest.raises(TypeError): - alpsLauncher.set_hostlist(["test",5]) + alpsLauncher.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): - alpsLauncher.set_hostlist([5]) + alpsLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): - alpsLauncher.set_hostlist(5) + alpsLauncher.launch_args.set_hostlist(5) def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) with pytest.raises(TypeError): - alpsLauncher.set_excluded_hosts(["test",5]) + alpsLauncher.launch_args.set_excluded_hosts(["test",5]) with pytest.raises(TypeError): - alpsLauncher.set_excluded_hosts([5]) + alpsLauncher.launch_args.set_excluded_hosts([5]) with pytest.raises(TypeError): - alpsLauncher.set_excluded_hosts(5) - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_nodes", (2,),id="set_nodes"), - pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), - pytest.param("set_node_feature", ("P100",),id="set_node_feature"), - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("set_task_map", ("task:map",), id="set_task_map"), - pytest.param("set_binding", ("bind",), id="set_binding"), - pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), - pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), - ], -) -def test_unimplimented_methods_throw_warning(caplog, method, params): - """Test methods not implemented throw warnings""" - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - try: - getattr(alpsLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "alps") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + alpsLauncher.launch_args.set_excluded_hosts(5) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py index d4b89a8f13..ec06e77114 100644 --- a/tests/temp_tests/test_settings/test_batchSettings.py +++ b/tests/temp_tests/test_settings/test_batchSettings.py @@ -3,27 +3,34 @@ import pytest @pytest.mark.parametrize( - "str,scheduler", + "scheduler_enum", [ - pytest.param("slurm", SchedulerType.SlurmScheduler, id="slurm"), - pytest.param("pbs", SchedulerType.PbsScheduler, id="dragon"), - pytest.param("lsf", SchedulerType.LsfScheduler, id="lsf"), + pytest.param(SchedulerType.SlurmScheduler, id="slurm"), + pytest.param(SchedulerType.PbsScheduler, id="dragon"), + pytest.param(SchedulerType.LsfScheduler, id="lsf"), ], ) -def test_create_from_scheduler_str(str, scheduler): - batchSettings = BatchSettings(batch_scheduler=str) - assert batchSettings.batch_scheduler == scheduler +def test_create_scheduler_settings(scheduler_enum): + bs_str = BatchSettings(batch_scheduler=scheduler_enum.value, scheduler_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + assert bs_str._batch_scheduler == scheduler_enum + # TODO need to test scheduler_args + assert bs_str._env_vars == {"ENV":"VAR"} + + bs_enum = BatchSettings(batch_scheduler=scheduler_enum, scheduler_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + assert bs_enum._batch_scheduler == scheduler_enum + # TODO need to test scheduler_args + assert bs_enum._env_vars == {"ENV":"VAR"} -def test_incorrect_env_var_type(): - with pytest.raises(TypeError): - _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, env_vars={"str": 2}) - with pytest.raises(TypeError): - _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, env_vars={"str": 2.0}) - with pytest.raises(TypeError): - _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, env_vars={"str": "str", "str": 2.0}) +def test_launcher_property(): + bs = BatchSettings(batch_scheduler="slurm") + assert bs.batch_scheduler == "slurm" -def test_incorrect_scheduler_arg_type(): - with pytest.raises(TypeError): - _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args={"str": [1,2]}) - with pytest.raises(TypeError): - _ = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args={"str": SchedulerType.SlurmScheduler}) \ No newline at end of file +def test_env_vars_property(): + bs = BatchSettings(batch_scheduler="slurm", env_vars={"ENV":"VAR"}) + assert bs.env_vars == {"ENV":"VAR"} + +def test_env_vars_property_deep_copy(): + bs = BatchSettings(batch_scheduler="slurm", env_vars={"ENV":"VAR"}) + copy_env_vars = bs.env_vars + copy_env_vars.update({"test":"no_update"}) + assert bs.env_vars == {"ENV":"VAR"} \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 43e803ef1a..bb95350c01 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -4,79 +4,15 @@ import pytest import logging -def test_launcher_str(): - """Ensure launcher_str returns appropriate value""" - dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert dragonLauncher.launcher_str() == LauncherType.DragonLauncher.value - -def test_set_reserved_launcher_args(): - """Ensure launcher_str returns appropriate value""" - dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert dragonLauncher.reserved_launch_args == set() - @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), - pytest.param("set_tasks_per_node", (2,),2,"tasks-per-node",id="set_tasks_per_node"), + pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), + pytest.param("set_tasks_per_node", (2,),"2","tasks-per-node",id="set_tasks_per_node"), ], ) def test_dragon_class_methods(function, value, flag, result): dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert isinstance(dragonLauncher.arg_translator,DragonArgTranslator) - getattr(dragonLauncher, function)(*value) - assert dragonLauncher.launcher_args[flag] == result - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("set_task_map", ("task:map",), id="set_task_map"), - pytest.param("set_hostlist", ("host_A",),id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),id="set_hostlist_list[str]"), - pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("host_A",),id="set_excluded_hosts_str"), - pytest.param("set_excluded_hosts", (["host_A","host_B"],),id="set_excluded_hosts_list[str]"), - pytest.param("set_cpu_bindings", (4,),id="set_cpu_bindings"), - pytest.param("set_cpu_bindings", ([4,4],),id="set_cpu_bindings_list[str]"), - pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), - pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), - pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), - pytest.param("set_walltime", ("10:00:00",),id="set_walltime"), - pytest.param("set_node_feature", ("P100",),id="set_node_feature"), - pytest.param("set_memory_per_node", ("1000",),id="set_memory_per_node"), - pytest.param("set_tasks", (2,),id="set_tasks"), - pytest.param("set_binding", ("bind",),id="set_tasks"), - pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), - pytest.param("format_launcher_args", (), id="format_launcher_args"), - pytest.param("format_env_vars", (), id="format_env_vars"), - pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - try: - getattr(dragonLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "dragon") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + assert isinstance(dragonLauncher._arg_translator,DragonArgTranslator) + getattr(dragonLauncher.launch_args, function)(*value) + assert dragonLauncher.launch_args._launch_args[flag] == result \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py index 9f774fe1f4..dbfe0bfbfb 100644 --- a/tests/temp_tests/test_settings/test_launchSettings.py +++ b/tests/temp_tests/test_settings/test_launchSettings.py @@ -4,138 +4,140 @@ import logging @pytest.mark.parametrize( - "str,launcher", + "launch_enum", [ - pytest.param("slurm", LauncherType.SlurmLauncher, id="slrum"), - pytest.param("dragon", LauncherType.DragonLauncher, id="dragon"), - pytest.param("pals", LauncherType.PalsLauncher, id="pals"), - pytest.param("alps", LauncherType.AlpsLauncher,id="alps"), - pytest.param("local", LauncherType.LocalLauncher,id="local"), - pytest.param("mpiexec", LauncherType.MpiexecLauncher,id="mpiexec"), - pytest.param("mpirun", LauncherType.MpirunLauncher,id="mpirun"), - pytest.param("orterun", LauncherType.OrterunLauncher,id="orterun"), - pytest.param("lsf", LauncherType.LsfLauncher,id="lsf"), + pytest.param(LauncherType.SlurmLauncher,id="slurm"), + pytest.param(LauncherType.DragonLauncher,id="dragon"), + pytest.param(LauncherType.PalsLauncher,id="pals"), + pytest.param(LauncherType.AlpsLauncher,id="alps"), + pytest.param(LauncherType.LocalLauncher,id="local"), + pytest.param(LauncherType.MpiexecLauncher,id="mpiexec"), + pytest.param(LauncherType.MpirunLauncher,id="mpirun"), + pytest.param(LauncherType.OrterunLauncher,id="orterun"), + pytest.param(LauncherType.LsfLauncher,id="lsf"), ], ) -def test_create_from_launcher_str(str, launcher): - launchSettings = LaunchSettings(launcher=str) - assert launchSettings.launcher == launcher - -def test_set_launch_args(): - launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) - launchSettings.set("str", "some-string") - launchSettings.set("nothing", None) - - assert "str" in launchSettings.launcher_args - assert launchSettings.launcher_args["str"] == "some-string" - - assert "nothing" in launchSettings.launcher_args - assert launchSettings.launcher_args["nothing"] is None - -@pytest.mark.parametrize( - "set_str,val,key", - [ - pytest.param("normal-key", "some-val", "normal-key", id="set string"), - pytest.param("--a-key", "a-value", "a-key", id="strip doulbe dashes"), - pytest.param("-b", "some-str", "b", id="strip single dashes"), - pytest.param(" c ", "some-val", "c", id="strip spaces"), - pytest.param(" --a-mess ", "5", "a-mess", id="strip everything"), - ], -) -def test_set_format_args(set_str, val, key): - launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) - launchSettings.set(set_str, val) - assert launchSettings.launcher_args[key] == val - -def test_set_raises_key_error(): - launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) - with pytest.raises(TypeError): - launchSettings.set(1, "test") - -def test_incorrect_env_var_type(): - with pytest.raises(TypeError): - _ = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars={"str": 2}) +def test_create_launch_settings(launch_enum): + ls_str = LaunchSettings(launcher=launch_enum.value, launch_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + assert ls_str._launcher == launch_enum + # TODO need to test launch_args + assert ls_str._env_vars == {"ENV":"VAR"} + + ls_enum = LaunchSettings(launcher=launch_enum, launch_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + assert ls_enum._launcher == launch_enum + # TODO need to test launch_args + assert ls_enum._env_vars == {"ENV":"VAR"} + +def test_launcher_property(): + ls = LaunchSettings(launcher="local") + assert ls.launcher == "local" + +def test_env_vars_property(): + ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) + assert ls.env_vars == {"ENV":"VAR"} + +def test_env_vars_property_deep_copy(): + ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) + copy_env_vars = ls.env_vars + copy_env_vars.update({"test":"no_update"}) + assert ls.env_vars == {"ENV":"VAR"} + +def test_update_env_vars(): + ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) + ls.update_env({"test":"no_update"}) + assert ls.env_vars == {"ENV":"VAR","test":"no_update"} + +def test_update_env_vars_errors(): + ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) with pytest.raises(TypeError): - _ = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars={"str": 2.0}) + ls.update_env({"test":1}) with pytest.raises(TypeError): - _ = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars={"str": "str", "str": 2.0}) - -def test_incorrect_launch_arg_type(): + ls.update_env({1:"test"}) with pytest.raises(TypeError): - _ = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args={"str": [1,2]}) + ls.update_env({1:1}) with pytest.raises(TypeError): - _ = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args={"str": LauncherType.LocalLauncher}) - -@pytest.mark.parametrize( - "launcher,key", - [ - pytest.param(LauncherType.SlurmLauncher, ("chdir",), id="slurm-chdir"), - pytest.param(LauncherType.SlurmLauncher, ("D",), id="slurm-D"), - pytest.param(LauncherType.LsfLauncher, ("chdir",), id="lsf-chdir"), - pytest.param(LauncherType.LsfLauncher, ("h",), id="lsf-h"), - pytest.param(LauncherType.MpiexecLauncher, ("wd",), id="mpiexec-wd"), - pytest.param(LauncherType.OrterunLauncher, ("wd",), id="orte-wd"), - pytest.param(LauncherType.MpirunLauncher, ("wd",), id="mpi-wd"), - pytest.param(LauncherType.MpiexecLauncher, ("wdir",), id="mpiexec-wdir"), - pytest.param(LauncherType.OrterunLauncher, ("wdir",), id="orte-wdir"), - pytest.param(LauncherType.MpirunLauncher, ("wdir",), id="mpi-wdir"), - ], -) -def test_prevent_set_reserved_launch_args(caplog, launcher, key): - """Test methods not implemented throw warnings""" - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - launchSettings = LaunchSettings(launcher=launcher) - try: - getattr(launchSettings, "set")(*key, None) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and "Could not set argument" in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{key}` is not " - "implemented at `warning` level" - ) - ) - -def test_log_overwrite_set_warning_message(caplog): - """Test methods not implemented throw warnings""" - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) - launchSettings.set("test", None) - try: - getattr(launchSettings, "set")("test", "overwritting") - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and "Overwritting argument" in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `test` will be " - "overwritten at `warning` level" - ) - ) \ No newline at end of file + # Make sure the first key and value do not assign + # and that the function is atomic + ls.update_env({"test":"test","test":1}) + assert ls.env_vars == {"ENV":"VAR"} + +# TODO need to test launch_args +# def test_set_launch_args(): +# ls = LaunchSettings(launcher="local", launch_args = {"init":"arg"}) +# assert ls.launch_args == {"init":"arg"} +# ls.launch_args = {"launch":"arg"} +# assert ls.launch_args == {"launch":"arg"} + +# @pytest.mark.parametrize( +# "launcher,key", +# [ +# pytest.param(LauncherType.SlurmLauncher, ("chdir",), id="slurm-chdir"), +# pytest.param(LauncherType.SlurmLauncher, ("D",), id="slurm-D"), +# pytest.param(LauncherType.LsfLauncher, ("chdir",), id="lsf-chdir"), +# pytest.param(LauncherType.LsfLauncher, ("h",), id="lsf-h"), +# pytest.param(LauncherType.MpiexecLauncher, ("wd",), id="mpiexec-wd"), +# pytest.param(LauncherType.OrterunLauncher, ("wd",), id="orte-wd"), +# pytest.param(LauncherType.MpirunLauncher, ("wd",), id="mpi-wd"), +# pytest.param(LauncherType.MpiexecLauncher, ("wdir",), id="mpiexec-wdir"), +# pytest.param(LauncherType.OrterunLauncher, ("wdir",), id="orte-wdir"), +# pytest.param(LauncherType.MpirunLauncher, ("wdir",), id="mpi-wdir"), +# ], +# ) +# def test_prevent_set_reserved_launch_args(caplog, launcher, key): +# """Test methods not implemented throw warnings""" +# from smartsim.settings.launchSettings import logger + +# prev_prop = logger.propagate +# logger.propagate = True + +# with caplog.at_level(logging.WARNING): +# caplog.clear() +# launchSettings = LaunchSettings(launcher=launcher) +# try: +# getattr(launchSettings, "set")(*key, None) +# finally: +# logger.propagate = prev_prop + +# for rec in caplog.records: +# if ( +# logging.WARNING <= rec.levelno < logging.ERROR +# and "Could not set argument" in rec.msg +# ): +# break +# else: +# pytest.fail( +# ( +# f"No message stating method `{key}` is not " +# "implemented at `warning` level" +# ) +# ) + +# def test_log_overwrite_set_warning_message(caplog): +# """Test methods not implemented throw warnings""" +# from smartsim.settings.launchSettings import logger + +# prev_prop = logger.propagate +# logger.propagate = True + +# with caplog.at_level(logging.WARNING): +# caplog.clear() +# launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) +# launchSettings.set("test", None) +# try: +# getattr(launchSettings, "set")("test", "overwritting") +# finally: +# logger.propagate = prev_prop + +# for rec in caplog.records: +# if ( +# logging.WARNING <= rec.levelno < logging.ERROR +# and "Overwritting argument" in rec.msg +# ): +# break +# else: +# pytest.fail( +# ( +# f"No message stating method `test` will be " +# "overwritten at `warning` level" +# ) +# ) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 403519f40e..68b86d2630 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -4,39 +4,30 @@ import pytest import logging -def test_launcher_str(): - """Ensure launcher_str returns appropriate value""" - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) - assert localLauncher.launcher_str() == LauncherType.LocalLauncher.value - -def test_set_reserved_launcher_args(): - """Ensure launcher_str returns appropriate value""" - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) - assert localLauncher.reserved_launch_args == set() - -def test_launch_args_input_mutation(): - # Tests that the run args passed in are not modified after initialization - key0, key1, key2 = "arg0", "arg1", "arg2" - val0, val1, val2 = "val0", "val1", "val2" - - default_launcher_args = { - key0: val0, - key1: val1, - key2: val2, - } - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args=default_launcher_args) - - # Confirm initial values are set - assert localLauncher.launcher_args[key0] == val0 - assert localLauncher.launcher_args[key1] == val1 - assert localLauncher.launcher_args[key2] == val2 - - # Update our common run arguments - val2_upd = f"not-{val2}" - default_launcher_args[key2] = val2_upd - - # Confirm previously created run settings are not changed - assert localLauncher.launcher_args[key2] == val2 +# TODO complete after launch args retrieval +# def test_launch_args_input_mutation(): +# # Tests that the run args passed in are not modified after initialization +# key0, key1, key2 = "arg0", "arg1", "arg2" +# val0, val1, val2 = "val0", "val1", "val2" + +# default_launcher_args = { +# key0: val0, +# key1: val1, +# key2: val2, +# } +# localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launch_args=default_launcher_args) + +# # Confirm initial values are set +# assert localLauncher.launcher_args[key0] == val0 +# assert localLauncher.launcher_args[key1] == val1 +# assert localLauncher.launcher_args[key2] == val2 + +# # Update our common run arguments +# val2_upd = f"not-{val2}" +# default_launcher_args[key2] = val2_upd + +# # Confirm previously created run settings are not changed +# assert localLauncher.launcher_args[key2] == val2 @pytest.mark.parametrize( "env_vars", @@ -54,10 +45,10 @@ def test_update_env(env_vars): assert len(localLauncher.env_vars) == len(env_vars.keys()) -def test_format_launcher_args(): - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launcher_args={"-np": 2}) - launcher_args = localLauncher.format_launcher_args() - assert launcher_args == ["-np", "2"] +def test_format_launch_args(): + localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launch_args={"-np": 2}) + launch_args = localLauncher.format_launch_args() + assert launch_args == ["np", "2"] @pytest.mark.parametrize( "env_vars", @@ -104,59 +95,5 @@ def test_format_env_vars(): "D": "12", } localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=env_vars) - assert isinstance(localLauncher.arg_translator, LocalArgTranslator) - assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_nodes", (2,), id="set_nodes"), - pytest.param("set_tasks", (2,), id="set_tasks"), - pytest.param("set_tasks_per_node", (3,), id="set_tasks_per_node"), - pytest.param("set_task_map", (3,), id="set_task_map"), - pytest.param("set_cpus_per_task", (4,), id="set_cpus_per_task"), - pytest.param("set_hostlist", ("hostlist",), id="set_hostlist"), - pytest.param("set_node_feature", ("P100",), id="set_node_feature"), - pytest.param( - "set_hostlist_from_file", ("~/hostfile",), id="set_hostlist_from_file" - ), - pytest.param("set_excluded_hosts", ("hostlist",), id="set_excluded_hosts"), - pytest.param("set_cpu_bindings", ([1, 2, 3],), id="set_cpu_bindings"), - pytest.param("set_memory_per_node", (16_000,), id="set_memory_per_node"), - pytest.param("set_verbose_launch", (False,), id="set_verbose_launch"), - pytest.param("set_quiet_launch", (True,), id="set_quiet_launch"), - pytest.param("set_walltime", ("00:55:00",), id="set_walltime"), - pytest.param("set_executable_broadcast", ("broad",), id="set_executable_broadcast"), - pytest.param("set_binding", ("packed:21",), id="set_binding"), - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), - pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) - try: - getattr(localLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and ("method" and "not supported" and "local") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + assert isinstance(localLauncher._arg_translator, LocalArgTranslator) + assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index c8285a146a..833c6d62cc 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -2,35 +2,24 @@ from smartsim.settings.translators.launch.lsf import JsrunArgTranslator from smartsim.settings.launchCommand import LauncherType import pytest -import logging - -def test_launcher_str(): - """Ensure launcher_str returns appropriate value""" - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - assert lsfLauncher.launcher_str() == LauncherType.LsfLauncher.value - -def test_set_reserved_launcher_args(): - """Ensure launcher_str returns appropriate value""" - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - assert lsfLauncher.reserved_launch_args == {"chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"} @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_tasks", (2,),2,"np",id="set_tasks"), + pytest.param("set_tasks", (2,),"2","np",id="set_tasks"), pytest.param("set_binding", ("packed:21",),"packed:21","bind",id="set_binding"), ], ) def test_lsf_class_methods(function, value, flag, result): lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) - getattr(lsfLauncher, function)(*value) - assert lsfLauncher.launcher_args[flag] == result + assert isinstance(lsfLauncher._arg_translator,JsrunArgTranslator) + getattr(lsfLauncher.launch_args, function)(*value) + assert lsfLauncher.launch_args._launch_args[flag] == result def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, env_vars=env_vars) - assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) + assert isinstance(lsfLauncher._arg_translator,JsrunArgTranslator) formatted = lsfLauncher.format_env_vars() assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] @@ -43,9 +32,9 @@ def test_launch_args(): "nrs": 10, "np": 100, } - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, launcher_args=launch_args) - assert isinstance(lsfLauncher.arg_translator,JsrunArgTranslator) - formatted = lsfLauncher.format_launcher_args() + lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, launch_args=launch_args) + assert isinstance(lsfLauncher._arg_translator,JsrunArgTranslator) + formatted = lsfLauncher.format_launch_args() result = [ "--latency_priority=gpu-gpu", "--immediate", @@ -54,58 +43,4 @@ def test_launch_args(): "--nrs=10", "--np=100", ] - assert formatted == result - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("set_task_map", ("task:map",), id="set_task_map"), - pytest.param("set_nodes", (2,), id="set_nodes"), - pytest.param("set_hostlist", ("host_A",),id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),id="set_hostlist_list[str]"), - pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("host_A",),id="set_excluded_hosts_str"), - pytest.param("set_excluded_hosts", (["host_A","host_B"],),id="set_excluded_hosts_list[str]"), - pytest.param("set_cpu_bindings", (4,),id="set_cpu_bindings"), - pytest.param("set_cpu_bindings", ([4,4],),id="set_cpu_bindings_list[str]"), - pytest.param("set_cpus_per_task", (2,),id="set_cpus_per_task"), - pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), - pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), - pytest.param("set_executable_broadcast", ("/tmp/some/path",),id="set_broadcast"), - pytest.param("set_walltime", ("10:00:00",),id="set_walltime"), - pytest.param("set_node_feature", ("P100",),id="set_node_feature"), - pytest.param("set_memory_per_node", ("1000",),id="set_memory_per_node"), - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("set_executable_broadcast", ("broad",),id="set_executable_broadcast"), - pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), - pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - try: - getattr(lsfLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "lsf") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + assert formatted == result \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py index a541e66b6f..25860176b5 100644 --- a/tests/temp_tests/test_settings/test_lsfScheduler.py +++ b/tests/temp_tests/test_settings/test_lsfScheduler.py @@ -3,71 +3,30 @@ import logging from smartsim.settings.batchCommand import SchedulerType -def test_scheduler_str(): - """Ensure launcher_str returns appropriate value""" - lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) - assert lsfScheduler.scheduler_str() == SchedulerType.LsfScheduler.value - @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),2,"nnodes",id="set_nodes"), + pytest.param("set_nodes", (2,),"2","nnodes",id="set_nodes"), pytest.param("set_walltime", ("10:00:00",),"10:00","W",id="set_walltime"), pytest.param("set_hostlist", ("host_A",),""'"host_A"'"","m",id="set_hostlist_str"), pytest.param("set_hostlist", (["host_A","host_B"],),""'"host_A host_B"'"","m",id="set_hostlist_list[str]"), - pytest.param("set_smts", (1,),1,"alloc_flags",id="set_smts"), + pytest.param("set_smts", (1,),"1","alloc_flags",id="set_smts"), pytest.param("set_project", ("project",),"project","P",id="set_project"), pytest.param("set_account", ("project",),"project","P",id="set_account"), - pytest.param("set_tasks", (2,),2,"n",id="set_tasks"), + pytest.param("set_tasks", (2,),"2","n",id="set_tasks"), pytest.param("set_queue", ("queue",),"queue","q",id="set_queue"), ], ) def test_update_env_initialized(function, value, flag, result): lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) - getattr(lsfScheduler, function)(*value) - assert lsfScheduler.scheduler_args[flag] == result + getattr(lsfScheduler.scheduler_args, function)(*value) + assert lsfScheduler.scheduler_args._scheduler_args[flag] == result def test_create_bsub(): batch_args = {"core_isolation": None} lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler, scheduler_args=batch_args) - lsfScheduler.set_nodes(1) - lsfScheduler.set_walltime("10:10:10") - lsfScheduler.set_queue("default") + lsfScheduler.scheduler_args.set_nodes(1) + lsfScheduler.scheduler_args.set_walltime("10:10:10") + lsfScheduler.scheduler_args.set_queue("default") args = lsfScheduler.format_batch_args() - assert args == ["-core_isolation", "-nnodes", "1", "-W", "10:10", "-q", "default"] - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_partition", (3,), id="set_tasks"), - pytest.param("set_cpus_per_task", (10,), id="set_smts"), - pytest.param("set_ncpus", (2,), id="set_ncpus"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) - try: - getattr(slurmScheduler, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "lsf") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + assert args == ["-core_isolation", "-nnodes", "1", "-W", "10:10", "-q", "default"] \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 3e61046939..b0b9d0ca6e 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -5,32 +5,6 @@ import itertools from smartsim.settings.launchCommand import LauncherType -@pytest.mark.parametrize( - "launcher", - [ - pytest.param(LauncherType.MpirunLauncher, id="format_launcher_args_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="format_launcher_args_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="format_launcher_args_orterun"), - ], -) -def test_launcher_str(launcher): - """Ensure launcher_str returns appropriate value""" - mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings.launcher_str() == LauncherType.PalsLauncher.value - -@pytest.mark.parametrize( - "launcher", - [ - pytest.param(LauncherType.MpirunLauncher, id="format_launcher_args_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="format_launcher_args_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="format_launcher_args_orterun"), - ], -) -def test_set_reserved_launcher_args(launcher): - """Ensure launcher_str returns appropriate value""" - mpiSettings = LaunchSettings(launcher=launcher) - assert mpiSettings.reserved_launch_args == {"wd", "wdir"} - @pytest.mark.parametrize( "l,function,value,result,flag", [ @@ -40,10 +14,10 @@ def test_set_reserved_launcher_args(launcher): ( pytest.param(l, "set_walltime", ("100",),"100","timeout",id="set_walltime"), pytest.param(l, "set_task_map", ("taskmap",),"taskmap","map-by",id="set_task_map"), - pytest.param(l, "set_cpus_per_task", (2,),2,"cpus-per-proc",id="set_cpus_per_task"), + pytest.param(l, "set_cpus_per_task", (2,),"2","cpus-per-proc",id="set_cpus_per_task"), pytest.param(l, "set_cpu_binding_type", ("4",),"4","bind-to",id="set_cpu_binding_type"), - pytest.param(l, "set_tasks_per_node", (4,),4,"npernode",id="set_tasks_per_node"), - pytest.param(l, "set_tasks", (4,),4,"n",id="set_tasks"), + pytest.param(l, "set_tasks_per_node", (4,),"4","npernode",id="set_tasks_per_node"), + pytest.param(l, "set_tasks", (4,),"4","n",id="set_tasks"), pytest.param(l, "set_executable_broadcast", ("broadcast",),"broadcast","preload-binary",id="set_executable_broadcast"), pytest.param(l, "set_hostlist", ("host_A",),"host_A","host",id="set_hostlist_str"), pytest.param(l, "set_hostlist", (["host_A","host_B"],),"host_A,host_B","host",id="set_hostlist_list[str]"), @@ -55,9 +29,9 @@ def test_set_reserved_launcher_args(launcher): ) def test_mpi_class_methods(l,function, value, flag, result): mpiSettings = LaunchSettings(launcher=l[0]) - assert isinstance(mpiSettings.arg_translator,l[1]) - getattr(mpiSettings, function)(*value) - assert mpiSettings.launcher_args[flag] == result + assert isinstance(mpiSettings._arg_translator,l[1]) + getattr(mpiSettings.launch_args, function)(*value) + assert mpiSettings.launch_args._launch_args[flag] == result @pytest.mark.parametrize( "launcher", @@ -89,10 +63,10 @@ def test_format_env_vars(launcher): ) def test_format_launcher_args(launcher): mpiSettings = LaunchSettings(launcher=launcher) - mpiSettings.set_cpus_per_task(1) - mpiSettings.set_tasks(2) - mpiSettings.set_hostlist(["node005", "node006"]) - formatted = mpiSettings.format_launcher_args() + mpiSettings.launch_args.set_cpus_per_task(1) + mpiSettings.launch_args.set_tasks(2) + mpiSettings.launch_args.set_hostlist(["node005", "node006"]) + formatted = mpiSettings.format_launch_args() result = ["--cpus-per-proc", "1", "--n", "2", "--host", "node005,node006"] assert formatted == result @@ -106,10 +80,10 @@ def test_format_launcher_args(launcher): ) def test_set_verbose_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) - mpiSettings.set_verbose_launch(True) - assert mpiSettings.launcher_args == {'verbose': None} - mpiSettings.set_verbose_launch(False) - assert mpiSettings.launcher_args == {} + mpiSettings.launch_args.set_verbose_launch(True) + assert mpiSettings.launch_args._launch_args == {'verbose': None} + mpiSettings.launch_args.set_verbose_launch(False) + assert mpiSettings.launch_args._launch_args == {} @pytest.mark.parametrize( "launcher", @@ -121,10 +95,10 @@ def test_set_verbose_launch(launcher): ) def test_set_quiet_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) - mpiSettings.set_quiet_launch(True) - assert mpiSettings.launcher_args == {'quiet': None} - mpiSettings.set_quiet_launch(False) - assert mpiSettings.launcher_args == {} + mpiSettings.launch_args.set_quiet_launch(True) + assert mpiSettings.launch_args._launch_args == {'quiet': None} + mpiSettings.launch_args.set_quiet_launch(False) + assert mpiSettings.launch_args._launch_args == {} @pytest.mark.parametrize( "launcher", @@ -138,67 +112,8 @@ def test_invalid_hostlist_format(launcher): """Test invalid hostlist formats""" mpiSettings = LaunchSettings(launcher=launcher) with pytest.raises(TypeError): - mpiSettings.set_hostlist(["test",5]) + mpiSettings.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): - mpiSettings.set_hostlist([5]) + mpiSettings.launch_args.set_hostlist([5]) with pytest.raises(TypeError): - mpiSettings.set_hostlist(5) - -@pytest.mark.parametrize( - "launcher", - [ - pytest.param(LauncherType.MpirunLauncher, id="launcher_str_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="launcher_str_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="launcher_str_orterun"), - ], -) -def test_launcher_str(launcher): - mpiLauncher = LaunchSettings(launcher=launcher) - assert mpiLauncher.launcher_str() == launcher.value - -@pytest.mark.parametrize( - "l,method,params", - [ - *itertools.chain.from_iterable( - ( - ( - pytest.param(l, "set_nodes", (1,), id="set_nodes"), - pytest.param(l, "set_excluded_hosts", ("hosts",), id="set_excluded_hosts"), - pytest.param(l, "set_cpu_bindings", (1,), id="set_cpu_bindings"), - pytest.param(l, "set_memory_per_node", (3000,), id="set_memory_per_node"), - pytest.param(l, "set_binding", ("bind",), id="set_binding"), - pytest.param(l, "set_node_feature", ("P100",), id="set_node_feature"), - pytest.param(l, "format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), - pytest.param(l, "set_het_group", ([1,2,3,4],), id="set_het_group"), - ) - for l in (LauncherType.MpirunLauncher, LauncherType.MpiexecLauncher, LauncherType.OrterunLauncher) - )) - ], -) -def test_unimplimented_methods_throw_warning(l, caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - mpiSettings = LaunchSettings(launcher=l) - try: - getattr(mpiSettings, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and l.value) in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + mpiSettings.launch_args.set_hostlist(5) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index f8d2b9b71a..f8b4908f5e 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -2,24 +2,13 @@ from smartsim.settings.translators.launch.pals import PalsMpiexecArgTranslator from smartsim.settings.launchCommand import LauncherType import pytest -import logging - -def test_launcher_str(): - """Ensure launcher_str returns appropriate value""" - palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) - assert palsLauncher.launcher_str() == LauncherType.PalsLauncher.value - -def test_set_reserved_launcher_args(): - """Ensure launcher_str returns appropriate value""" - palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) - assert palsLauncher.reserved_launch_args == set() @pytest.mark.parametrize( "function,value,result,flag", [ pytest.param("set_cpu_binding_type", ("bind",),"bind","bind-to",id="set_cpu_binding_type"), - pytest.param("set_tasks", (2,),2,"np",id="set_tasks"), - pytest.param("set_tasks_per_node", (2,),2,"ppn",id="set_tasks_per_node"), + pytest.param("set_tasks", (2,),"2","np",id="set_tasks"), + pytest.param("set_tasks_per_node", (2,),"2","ppn",id="set_tasks_per_node"), pytest.param("set_hostlist", ("host_A",),"host_A","hosts",id="set_hostlist_str"), pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hosts",id="set_hostlist_list[str]"), pytest.param("set_executable_broadcast", ("broadcast",),"broadcast","transfer",id="set_executable_broadcast"), @@ -27,10 +16,9 @@ def test_set_reserved_launcher_args(): ) def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) - getattr(palsLauncher, function)(*value) - assert isinstance(palsLauncher.arg_translator,PalsMpiexecArgTranslator) - assert palsLauncher.launcher_args[flag] == result - assert palsLauncher.format_launcher_args() == ["--" + flag, str(result)] + getattr(palsLauncher.launch_args, function)(*value) + assert palsLauncher.launch_args._launch_args[flag] == result + assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] def test_format_env_vars(): env_vars = {"FOO_VERSION": "3.14", "PATH": None, "LD_LIBRARY_PATH": None} @@ -43,56 +31,8 @@ def test_invalid_hostlist_format(): """Test invalid hostlist formats""" palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) with pytest.raises(TypeError): - palsLauncher.set_hostlist(["test",5]) + palsLauncher.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): - palsLauncher.set_hostlist([5]) + palsLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): - palsLauncher.set_hostlist(5) - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_cpu_bindings", ("bind",), id="set_cpu_bindings"), - pytest.param("set_binding", ("bind",), id="set_binding"), - pytest.param("set_nodes", (2,), id="set_nodes"), - pytest.param("set_task_map", ("task:map",), id="set_task_map"), - pytest.param("set_cpus_per_task", ("task:map",), id="set_cpus_per_task"), - pytest.param("set_quiet_launch", ("task:map",), id="set_quiet_launch"), - pytest.param("set_walltime", ("task:map",), id="set_walltime"), - pytest.param("set_node_feature", ("P100",),id="set_node_feature"), - pytest.param("set_hostlist_from_file", ("./file",),id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("./file",),id="set_excluded_hosts"), - pytest.param("set_memory_per_node", ("8000",),id="set_memory_per_node"), - pytest.param("set_verbose_launch", (True,),id="set_verbose_launch"), - pytest.param("set_quiet_launch", (True,),id="set_quiet_launch"), - pytest.param("format_comma_sep_env_vars", (), id="format_comma_sep_env_vars"), - pytest.param("set_het_group", ([1,2,3,4],), id="set_het_group"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) - try: - getattr(palsLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and ("not supported" and "pals") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + palsLauncher.launch_args.set_hostlist(5) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index ce307806de..0c7e33522a 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -2,79 +2,35 @@ from smartsim.settings.translators.batch.pbs import QsubBatchArgTranslator from smartsim.settings.batchCommand import SchedulerType import pytest -import logging - -def test_scheduler_str(): - """Ensure launcher_str returns appropriate value""" - pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) - assert pbsScheduler.scheduler_str() == SchedulerType.PbsScheduler.value @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), pytest.param("set_walltime", ("10:00:00",),"10:00:00","walltime",id="set_walltime"), pytest.param("set_account", ("account",),"account","A",id="set_account"), pytest.param("set_queue", ("queue",),"queue","q",id="set_queue"), - pytest.param("set_ncpus", (2,),2,"ppn",id="set_ncpus"), + pytest.param("set_ncpus", (2,),"2","ppn",id="set_ncpus"), pytest.param("set_hostlist", ("host_A",),"host_A","hostname",id="set_hostlist_str"), pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hostname",id="set_hostlist_list[str]"), ], ) def test_update_env_initialized(function, value, flag, result): pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) - getattr(pbsScheduler, function)(*value) - assert pbsScheduler.scheduler_args[flag] == result + getattr(pbsScheduler.scheduler_args, function)(*value) + assert pbsScheduler.scheduler_args._scheduler_args[flag] == result def test_create_pbs_batch(): pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) - pbsScheduler.set_nodes(1) - pbsScheduler.set_walltime("10:00:00") - pbsScheduler.set_queue("default") - pbsScheduler.set_account("myproject") - pbsScheduler.set_ncpus(10) + pbsScheduler.scheduler_args.set_nodes(1) + pbsScheduler.scheduler_args.set_walltime("10:00:00") + pbsScheduler.scheduler_args.set_queue("default") + pbsScheduler.scheduler_args.set_account("myproject") + pbsScheduler.scheduler_args.set_ncpus(10) args = pbsScheduler.format_batch_args() assert args == [ "-l", "nodes=1:ncpus=10", "-l", "walltime=10:00:00", "-q", "default", "-A", "myproject", - ] - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_tasks", (3,), id="set_tasks"), - pytest.param("set_smts", ("smts",), id="set_smts"), - pytest.param("set_cpus_per_task", (2,), id="set_cpus_per_task"), - pytest.param("set_project", ("project",), id="set_project"), - pytest.param("set_partition", ("project",), id="set_partition"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) - try: - getattr(pbsScheduler, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "pbs") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + ] \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 76778f19c1..abc1f2798f 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -4,58 +4,45 @@ import pytest import logging -def test_launcher_str(): - """Ensure launcher_str returns appropriate value""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert slurmLauncher.launcher_str() == LauncherType.SlurmLauncher.value - -def test_set_reserved_launcher_args(): - """Ensure launcher_str returns appropriate value""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert slurmLauncher.reserved_launch_args == {"chdir", "D"} - @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), pytest.param("set_hostlist", ("host_A",),"host_A","nodelist",id="set_hostlist_str"), pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","nodelist",id="set_hostlist_list[str]"), pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","nodefile",id="set_hostlist_from_file"), pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude",id="set_excluded_hosts_str"), pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude",id="set_excluded_hosts_list[str]"), - pytest.param("set_cpus_per_task", (4,),4,"cpus-per-task",id="set_cpus_per_task"), - pytest.param("set_tasks", (4,),4,"ntasks",id="set_tasks"), - pytest.param("set_tasks_per_node", (4,),4,"ntasks-per-node",id="set_tasks_per_node"), + pytest.param("set_cpus_per_task", (4,),"4","cpus-per-task",id="set_cpus_per_task"), + pytest.param("set_tasks", (4,),"4","ntasks",id="set_tasks"), + pytest.param("set_tasks_per_node", (4,),"4","ntasks-per-node",id="set_tasks_per_node"), pytest.param("set_cpu_bindings", (4,),"map_cpu:4","cpu_bind",id="set_cpu_bindings"), pytest.param("set_cpu_bindings", ([4,4],),"map_cpu:4,4","cpu_bind",id="set_cpu_bindings_list[str]"), pytest.param("set_memory_per_node", (8000,),"8000M","mem",id="set_memory_per_node"), pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_broadcast"), pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), pytest.param("set_walltime", ("10:00:00",),"10:00:00","time",id="set_walltime"), - pytest.param("set_verbose_launch", (True,),None,"verbose",id="set_walltime"), ], ) def test_slurm_class_methods(function, value, flag, result): slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) - getattr(slurmLauncher, function)(*value) - assert slurmLauncher.launcher_args[flag] == result + assert isinstance(slurmLauncher.launch_args,SlurmArgTranslator) + getattr(slurmLauncher.launch_args, function)(*value) + assert slurmLauncher.launch_args._launch_args[flag] == result def test_set_verbose_launch(): - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) - slurmLauncher.set_verbose_launch(True) - assert slurmLauncher.launcher_args == {'verbose': None} - slurmLauncher.set_verbose_launch(False) - assert slurmLauncher.launcher_args == {} + ls = LaunchSettings(launcher=LauncherType.SlurmLauncher) + ls.launch_args.set_verbose_launch(True) + assert ls.launch_args._launch_args == {'verbose': None} + ls.launch_args.set_verbose_launch(False) + assert ls.launch_args._launch_args == {} def test_set_quiet_launch(): - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) - slurmLauncher.set_quiet_launch(True) - assert slurmLauncher.launcher_args == {'quiet': None} - slurmLauncher.set_quiet_launch(False) - assert slurmLauncher.launcher_args == {} + ls = LaunchSettings(launcher=LauncherType.SlurmLauncher) + ls.launch_args.set_quiet_launch(True) + assert ls.launch_args._launch_args == {'quiet': None} + ls.launch_args.set_quiet_launch(False) + assert ls.launch_args._launch_args == {} def test_format_env_vars(): """Test format_env_vars runs correctly""" @@ -64,12 +51,11 @@ def test_format_env_vars(): "LOGGING": "verbose", "SSKEYIN": "name_0,name_1", } - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) - assert isinstance(slurmLauncher.arg_translator,SlurmArgTranslator) - formatted = slurmLauncher.format_env_vars() - assert "OMP_NUM_THREADS=20" in formatted - assert "LOGGING=verbose" in formatted - assert all("SSKEYIN" not in x for x in formatted) + ls = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) + ls_format = ls.format_env_vars() + assert "OMP_NUM_THREADS=20" in ls_format + assert "LOGGING=verbose" in ls_format + assert all("SSKEYIN" not in x for x in ls_format) def test_catch_existing_env_var(caplog, monkeypatch): slurmSettings = LaunchSettings( @@ -113,27 +99,27 @@ def test_format_comma_sep_env_vars(): assert "SSKEYIN=name_0,name_1" in comma_separated_formatted def test_slurmSettings_settings(): - """Test format_launcher_args runs correctly""" + """Test format_launch_args runs correctly""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - slurmLauncher.set_nodes(5) - slurmLauncher.set_cpus_per_task(2) - slurmLauncher.set_tasks(100) - slurmLauncher.set_tasks_per_node(20) - formatted = slurmLauncher.format_launcher_args() + slurmLauncher.launch_args.set_nodes(5) + slurmLauncher.launch_args.set_cpus_per_task(2) + slurmLauncher.launch_args.set_tasks(100) + slurmLauncher.launch_args.set_tasks_per_node(20) + formatted = slurmLauncher.format_launch_args() result = ["--nodes=5", "--cpus-per-task=2", "--ntasks=100", "--ntasks-per-node=20"] assert formatted == result -def test_slurmSettings_launcher_args(): +def test_slurmSettings_launch_args(): """Test the possible user overrides through run_args""" - launcher_args = { + launch_args = { "account": "A3123", "exclusive": None, "C": "P100", # test single letter variables "nodes": 10, "ntasks": 100, } - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, launcher_args=launcher_args) - formatted = slurmLauncher.format_launcher_args() + slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, launch_args=launch_args) + formatted = slurmLauncher.format_launch_args() result = [ "--account=A3123", "--exclusive", @@ -148,88 +134,51 @@ def test_invalid_hostlist_format(): """Test invalid hostlist formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(TypeError): - slurmLauncher.set_hostlist(["test",5]) + slurmLauncher.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): - slurmLauncher.set_hostlist([5]) + slurmLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): - slurmLauncher.set_hostlist(5) + slurmLauncher.launch_args.set_hostlist(5) def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(TypeError): - slurmLauncher.set_excluded_hosts(["test",5]) + slurmLauncher.launch_args.set_excluded_hosts(["test",5]) with pytest.raises(TypeError): - slurmLauncher.set_excluded_hosts([5]) + slurmLauncher.launch_args.set_excluded_hosts([5]) with pytest.raises(TypeError): - slurmLauncher.set_excluded_hosts(5) + slurmLauncher.launch_args.set_excluded_hosts(5) def test_invalid_node_feature_format(): """Test invalid node feature formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(TypeError): - slurmLauncher.set_node_feature(["test",5]) + slurmLauncher.launch_args.set_node_feature(["test",5]) with pytest.raises(TypeError): - slurmLauncher.set_node_feature([5]) + slurmLauncher.launch_args.set_node_feature([5]) with pytest.raises(TypeError): - slurmLauncher.set_node_feature(5) + slurmLauncher.launch_args.set_node_feature(5) def test_invalid_walltime_format(): """Test invalid walltime formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) with pytest.raises(ValueError): - slurmLauncher.set_walltime("11:11") + slurmLauncher.launch_args.set_walltime("11:11") with pytest.raises(ValueError): - slurmLauncher.set_walltime("ss:ss:ss") + slurmLauncher.launch_args.set_walltime("ss:ss:ss") with pytest.raises(ValueError): - slurmLauncher.set_walltime("11:ss:ss") + slurmLauncher.launch_args.set_walltime("11:ss:ss") with pytest.raises(ValueError): - slurmLauncher.set_walltime("0s:ss:ss") - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_cpu_binding_type", ("bind",), id="set_cpu_binding_type"), - pytest.param("set_task_map", ("task:map",), id="set_task_map"), - pytest.param("set_binding", ("bind",), id="set_binding"), - ], -) -def test_unimplimented_methods_throw_warning(caplog, method, params): - """Test methods not implemented throw warnings""" - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - try: - getattr(slurmLauncher, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "slurm") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) + slurmLauncher.launch_args.set_walltime("0s:ss:ss") def test_set_het_groups(monkeypatch): """Test ability to set one or more het groups to run setting""" monkeypatch.setenv("SLURM_HET_SIZE", "4") slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - slurmLauncher.set_het_group([1]) - assert slurmLauncher.launcher_args["het-group"] == "1" - slurmLauncher.set_het_group([3, 2]) - assert slurmLauncher.launcher_args["het-group"] == "3,2" + slurmLauncher.launch_args.set_het_group([1]) + assert slurmLauncher._arg_translator._launch_args["het-group"] == "1" + slurmLauncher.launch_args.set_het_group([3, 2]) + assert slurmLauncher._arg_translator._launch_args["het-group"] == "3,2" with pytest.raises(ValueError): - slurmLauncher.set_het_group([4]) \ No newline at end of file + slurmLauncher.launch_args.set_het_group([4]) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index 1c53552ad2..92a0d530f3 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -2,35 +2,29 @@ from smartsim.settings.translators.batch.slurm import SlurmBatchArgTranslator from smartsim.settings.batchCommand import SchedulerType import pytest -import logging - -def test_scheduler_str(): - """Ensure launcher_str returns appropriate value""" - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) - assert slurmScheduler.scheduler_str() == SchedulerType.SlurmScheduler.value @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),2,"nodes",id="set_nodes"), + pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), pytest.param("set_walltime", ("10:00:00",),"10:00:00","time",id="set_walltime"), pytest.param("set_account", ("account",),"account","account",id="set_account"), pytest.param("set_partition", ("partition",),"partition","partition",id="set_partition"), pytest.param("set_queue", ("partition",),"partition","partition",id="set_queue"), - pytest.param("set_cpus_per_task", (2,),2,"cpus-per-task",id="set_cpus_per_task"), + pytest.param("set_cpus_per_task", (2,),"2","cpus-per-task",id="set_cpus_per_task"), pytest.param("set_hostlist", ("host_A",),"host_A","nodelist",id="set_hostlist_str"), pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","nodelist",id="set_hostlist_list[str]"), ], ) -def test_update_env_initialized(function, value, flag, result): +def test_sbatch_class_methods(function, value, flag, result): slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) - getattr(slurmScheduler, function)(*value) - assert slurmScheduler.scheduler_args[flag] == result + getattr(slurmScheduler.scheduler_args, function)(*value) + assert slurmScheduler.scheduler_args._scheduler_args[flag] == result def test_create_sbatch(): batch_args = {"exclusive": None, "oversubscribe": None} slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args=batch_args) - assert isinstance(slurmScheduler.arg_translator, SlurmBatchArgTranslator) + assert isinstance(slurmScheduler._arg_translator, SlurmBatchArgTranslator) #assert slurmScheduler.batch_args["partition"] == "default" args = slurmScheduler.format_batch_args() assert args == [ @@ -51,16 +45,16 @@ def test_launch_args_input_mutation(): slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args=default_scheduler_args) # Confirm initial values are set - assert slurmScheduler.scheduler_args[key0] == val0 - assert slurmScheduler.scheduler_args[key1] == val1 - assert slurmScheduler.scheduler_args[key2] == val2 + assert slurmScheduler.scheduler_args._scheduler_args[key0] == val0 + assert slurmScheduler.scheduler_args._scheduler_args[key1] == val1 + assert slurmScheduler.scheduler_args._scheduler_args[key2] == val2 # Update our common run arguments val2_upd = f"not-{val2}" default_scheduler_args[key2] = val2_upd # Confirm previously created run settings are not changed - assert slurmScheduler.scheduler_args[key2] == val2 + assert slurmScheduler.scheduler_args._scheduler_args[key2] == val2 def test_sbatch_settings(): scheduler_args = {"nodes": 1, "time": "10:00:00", "account": "A3123"} @@ -72,46 +66,10 @@ def test_sbatch_settings(): def test_sbatch_manual(): slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) - slurmScheduler.set_nodes(5) - slurmScheduler.set_account("A3531") - slurmScheduler.set_walltime("10:00:00") + slurmScheduler.scheduler_args.set_nodes(5) + slurmScheduler.scheduler_args.set_account("A3531") + slurmScheduler.scheduler_args.set_walltime("10:00:00") formatted = slurmScheduler.format_batch_args() + print(f"here: {formatted}") result = ["--nodes=5", "--account=A3531", "--time=10:00:00"] - assert formatted == result - -@pytest.mark.parametrize( - "method,params", - [ - pytest.param("set_tasks", (3,), id="set_tasks"), - pytest.param("set_smts", ("smts",), id="set_smts"), - pytest.param("set_ncpus", (2,), id="set_ncpus"), - pytest.param("set_project", ("project",), id="set_project"), - ], -) -def test_unimplimented_setters_throw_warning(caplog, method, params): - from smartsim.settings.launchSettings import logger - - prev_prop = logger.propagate - logger.propagate = True - - with caplog.at_level(logging.WARNING): - caplog.clear() - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) - try: - getattr(slurmScheduler, method)(*params) - finally: - logger.propagate = prev_prop - - for rec in caplog.records: - if ( - logging.WARNING <= rec.levelno < logging.ERROR - and (method and "not supported" and "slurm") in rec.msg - ): - break - else: - pytest.fail( - ( - f"No message stating method `{method}` is not " - "implemented at `warning` level" - ) - ) \ No newline at end of file + assert formatted == result \ No newline at end of file From 26ca1ea44ff91d09e157af10aa7f6dc960c8b349 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 5 Jun 2024 01:59:24 -0500 Subject: [PATCH 28/43] mock imports --- smartsim/_core/launcher/step/pbsStep.py | 4 +--- smartsim/_core/launcher/step/slurmStep.py | 7 +------ smartsim/_core/launcher/step/step.py | 5 +---- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/smartsim/_core/launcher/step/pbsStep.py b/smartsim/_core/launcher/step/pbsStep.py index 81f5f73336..5d484031f0 100644 --- a/smartsim/_core/launcher/step/pbsStep.py +++ b/smartsim/_core/launcher/step/pbsStep.py @@ -28,9 +28,7 @@ from ....entity import DBNode, Model from ....log import get_logger -# from ....settings import QsubBatchSettings -# Temp mock imports -class QsubBatchSettings: pass +from ....settings import QsubBatchSettings from .step import Step diff --git a/smartsim/_core/launcher/step/slurmStep.py b/smartsim/_core/launcher/step/slurmStep.py index 3b47cc7d94..a402d1e03d 100644 --- a/smartsim/_core/launcher/step/slurmStep.py +++ b/smartsim/_core/launcher/step/slurmStep.py @@ -32,12 +32,7 @@ from ....entity import DBNode, Ensemble, Model from ....error import AllocationError from ....log import get_logger -# from ....settings import RunSettings, SbatchSettings, Singularity, SrunSettings -# Temp mock imports -class RunSettings: pass -class SbatchSettings: pass -class Singularity: pass -class SrunSettings: pass +from ....settings import RunSettings, SbatchSettings, Singularity, SrunSettings from .step import Step diff --git a/smartsim/_core/launcher/step/step.py b/smartsim/_core/launcher/step/step.py index 9cdd3721b4..c8cc9c5789 100644 --- a/smartsim/_core/launcher/step/step.py +++ b/smartsim/_core/launcher/step/step.py @@ -39,10 +39,7 @@ from ....entity import DBNode, Ensemble, Model from ....log import get_logger -# from ....settings.base import RunSettings, SettingsBase -# Temp mock imports -class RunSettings: pass -class SettingsBase: pass +from ....settings import RunSettings, SettingsBase from ...utils.helpers import encode_cmd, get_base_36_repr from ..colocated import write_colocated_launch_script From a74349174c700dfe9b76c427e8f2079848cc58e5 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 5 Jun 2024 02:09:16 -0500 Subject: [PATCH 29/43] chnage testing --- smartsim/_core/launcher/__init__.py | 2 +- .../_core/launcher/dragon/dragonLauncher.py | 14 +++--- smartsim/_core/launcher/launcher.py | 4 +- smartsim/_core/launcher/lsf/lsfLauncher.py | 1 - smartsim/_core/launcher/pbs/pbsLauncher.py | 1 - .../_core/launcher/slurm/slurmLauncher.py | 1 - smartsim/_core/launcher/step/alpsStep.py | 2 +- smartsim/_core/launcher/step/dragonStep.py | 1 - smartsim/_core/launcher/step/pbsStep.py | 1 - smartsim/_core/launcher/step/slurmStep.py | 1 - smartsim/database/orchestrator.py | 48 +++++++------------ smartsim/entity/dbnode.py | 4 +- smartsim/entity/ensemble.py | 6 +-- smartsim/entity/entity.py | 4 -- smartsim/settings/__init__.py | 6 ++- 15 files changed, 35 insertions(+), 61 deletions(-) diff --git a/smartsim/_core/launcher/__init__.py b/smartsim/_core/launcher/__init__.py index a50a7c3ce8..d789096415 100644 --- a/smartsim/_core/launcher/__init__.py +++ b/smartsim/_core/launcher/__init__.py @@ -38,4 +38,4 @@ "LSFLauncher", "PBSLauncher", "SlurmLauncher", -] \ No newline at end of file +] diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index fee78160c6..23b2b3a046 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -39,7 +39,6 @@ SbatchSettings, SettingsBase, ) - from ....status import SmartSimStatus from ...schemas import ( DragonRunRequest, @@ -90,17 +89,16 @@ def cleanup(self) -> None: self._connector.cleanup() # RunSettings types supported by this launcher - # t.Type[SettingsBase] @property def supported_rs(self) -> t.Dict[t.Type[SettingsBase], t.Type[Step]]: # RunSettings types supported by this launcher pass - # return { - # DragonRunSettings: DragonStep, - # SbatchSettings: DragonBatchStep, - # QsubBatchSettings: DragonBatchStep, - # RunSettings: LocalStep, - # } + return { + DragonRunSettings: DragonStep, + SbatchSettings: DragonBatchStep, + QsubBatchSettings: DragonBatchStep, + RunSettings: LocalStep, + } def add_step_to_mapping_table(self, name: str, step_map: StepMap) -> None: super().add_step_to_mapping_table(name, step_map) diff --git a/smartsim/_core/launcher/launcher.py b/smartsim/_core/launcher/launcher.py index a8c3d12ed4..37e1447137 100644 --- a/smartsim/_core/launcher/launcher.py +++ b/smartsim/_core/launcher/launcher.py @@ -30,9 +30,7 @@ from ..._core.launcher.stepMapping import StepMap from ...entity import SmartSimEntity from ...error import AllocationError, LauncherError, SSUnsupportedError -# from ...settings import SettingsBase -# Mock imports -class SettingsBase: pass +from ...settings import SettingsBase from .step import Step from .stepInfo import StepInfo, UnmanagedStepInfo diff --git a/smartsim/_core/launcher/lsf/lsfLauncher.py b/smartsim/_core/launcher/lsf/lsfLauncher.py index d91f969e3e..e0ad808ed8 100644 --- a/smartsim/_core/launcher/lsf/lsfLauncher.py +++ b/smartsim/_core/launcher/lsf/lsfLauncher.py @@ -38,7 +38,6 @@ RunSettings, SettingsBase, ) - from ....status import SmartSimStatus from ...config import CONFIG from ..launcher import WLMLauncher diff --git a/smartsim/_core/launcher/pbs/pbsLauncher.py b/smartsim/_core/launcher/pbs/pbsLauncher.py index db6a768f51..8c2099a8bc 100644 --- a/smartsim/_core/launcher/pbs/pbsLauncher.py +++ b/smartsim/_core/launcher/pbs/pbsLauncher.py @@ -39,7 +39,6 @@ RunSettings, SettingsBase, ) - from ....status import SmartSimStatus from ...config import CONFIG from ..launcher import WLMLauncher diff --git a/smartsim/_core/launcher/slurm/slurmLauncher.py b/smartsim/_core/launcher/slurm/slurmLauncher.py index a1742415c4..2e41023919 100644 --- a/smartsim/_core/launcher/slurm/slurmLauncher.py +++ b/smartsim/_core/launcher/slurm/slurmLauncher.py @@ -40,7 +40,6 @@ SettingsBase, SrunSettings, ) - from ....status import SmartSimStatus from ...config import CONFIG from ..launcher import WLMLauncher diff --git a/smartsim/_core/launcher/step/alpsStep.py b/smartsim/_core/launcher/step/alpsStep.py index 6c6ee89fc7..464f723a76 100644 --- a/smartsim/_core/launcher/step/alpsStep.py +++ b/smartsim/_core/launcher/step/alpsStep.py @@ -88,7 +88,7 @@ def get_launch_cmd(self) -> t.List[str]: launch_script_path = self.get_colocated_launch_script() aprun_cmd.extend([bash, launch_script_path]) - if isinstance(self.run_settings.container, Singularity): #changed from Singularity to str + if isinstance(self.run_settings.container, Singularity): # pylint: disable-next=protected-access aprun_cmd += self.run_settings.container._container_cmds(self.cwd) diff --git a/smartsim/_core/launcher/step/dragonStep.py b/smartsim/_core/launcher/step/dragonStep.py index 6380be5fa6..036a9e5654 100644 --- a/smartsim/_core/launcher/step/dragonStep.py +++ b/smartsim/_core/launcher/step/dragonStep.py @@ -39,7 +39,6 @@ SbatchSettings, Singularity, ) - from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/pbsStep.py b/smartsim/_core/launcher/step/pbsStep.py index 5d484031f0..9177dd6d56 100644 --- a/smartsim/_core/launcher/step/pbsStep.py +++ b/smartsim/_core/launcher/step/pbsStep.py @@ -29,7 +29,6 @@ from ....entity import DBNode, Model from ....log import get_logger from ....settings import QsubBatchSettings - from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/slurmStep.py b/smartsim/_core/launcher/step/slurmStep.py index a402d1e03d..5711a56942 100644 --- a/smartsim/_core/launcher/step/slurmStep.py +++ b/smartsim/_core/launcher/step/slurmStep.py @@ -33,7 +33,6 @@ from ....error import AllocationError from ....log import get_logger from ....settings import RunSettings, SbatchSettings, Singularity, SrunSettings - from .step import Step logger = get_logger(__name__) diff --git a/smartsim/database/orchestrator.py b/smartsim/database/orchestrator.py index 6a40fe49b5..17544cdca8 100644 --- a/smartsim/database/orchestrator.py +++ b/smartsim/database/orchestrator.py @@ -50,38 +50,22 @@ ) from ..log import get_logger from ..servertype import CLUSTERED, STANDALONE -# from ..settings import ( -# AprunSettings, -# BsubBatchSettings, -# JsrunSettings, -# MpiexecSettings, -# MpirunSettings, -# OrterunSettings, -# PalsMpiexecSettings, -# QsubBatchSettings, -# SbatchSettings, -# SrunSettings, -# ) -# Mock imports -class AprunSettings: pass -class BsubBatchSettings: pass -class JsrunSettings: pass -class MpiexecSettings: pass -class OrterunSettings: pass -class PalsMpiexecSettings: pass -class QsubBatchSettings: pass -class SbatchSettings: pass -class SrunSettings: pass - -# from ..settings.base import BatchSettings, RunSettings -# Mock imports -class BatchSettings: pass -class RunSettings: pass - -# from ..settings.settings import create_batch_settings, create_run_settings -# Mock functions -def create_batch_settings() -> None: ... -def create_run_settings() -> None: ... +from ..settings import ( + AprunSettings, + BsubBatchSettings, + JsrunSettings, + MpiexecSettings, + MpirunSettings, + OrterunSettings, + PalsMpiexecSettings, + QsubBatchSettings, + SbatchSettings, + SrunSettings, +) + +from ..settings import BatchSettings, RunSettings + +from ..settings import create_batch_settings, create_run_settings from ..wlm import detect_launcher logger = get_logger(__name__) diff --git a/smartsim/entity/dbnode.py b/smartsim/entity/dbnode.py index afeed68dfc..a43153e4bb 100644 --- a/smartsim/entity/dbnode.py +++ b/smartsim/entity/dbnode.py @@ -37,7 +37,7 @@ from .._core.utils.helpers import expand_exe_path from ..error import SSDBFilesNotParseable from ..log import get_logger -# from ..settings.base import RunSettings +from ..settings import RunSettings from .entity import SmartSimEntity logger = get_logger(__name__) @@ -58,7 +58,7 @@ def __init__( path: str, exe: str, exe_args: t.List[str], - # run_settings: RunSettings, + run_settings: RunSettings, ports: t.List[int], output_files: t.List[str], db_identifier: str = "", diff --git a/smartsim/entity/ensemble.py b/smartsim/entity/ensemble.py index 050a545620..5c807c4d72 100644 --- a/smartsim/entity/ensemble.py +++ b/smartsim/entity/ensemble.py @@ -40,7 +40,7 @@ UserStrategyError, ) from ..log import get_logger -# from ..settings.base import BatchSettings, RunSettings +from ..settings import BatchSettings, RunSettings from .dbobject import DBModel, DBScript from .entity import SmartSimEntity from .entityList import EntityList @@ -67,8 +67,8 @@ def __init__( exe_args: t.Optional[t.List[str]] = None, path: t.Optional[str] = getcwd(), params_as_args: t.Optional[t.List[str]] = None, - # batch_settings: t.Optional[BatchSettings] = None, - # run_settings: t.Optional[RunSettings] = None, + batch_settings: t.Optional[BatchSettings] = None, + run_settings: t.Optional[RunSettings] = None, perm_strat: str = "all_perm", **kwargs: t.Any, ) -> None: diff --git a/smartsim/entity/entity.py b/smartsim/entity/entity.py index 6d08efbf60..2f4b651f99 100644 --- a/smartsim/entity/entity.py +++ b/smartsim/entity/entity.py @@ -26,10 +26,6 @@ import typing as t -# if t.TYPE_CHECKING: - # pylint: disable-next=unused-import - # import smartsim.settings.base - class TelemetryConfiguration: """A base class for configuraing telemetry production behavior on diff --git a/smartsim/settings/__init__.py b/smartsim/settings/__init__.py index 33b54de91a..2c52a3bedf 100644 --- a/smartsim/settings/__init__.py +++ b/smartsim/settings/__init__.py @@ -47,4 +47,8 @@ class MpiexecSettings: pass class JsrunSettings: pass class BsubBatchSettings: pass class PalsMpiexecSettings: pass -class SrunSettings: pass \ No newline at end of file +class SrunSettings: pass +class Container: pass + +def create_batch_settings() -> None: ... +def create_run_settings() -> None: ... \ No newline at end of file From 9fdfca73dd839737946a3dd7aa2654e8fabb9e79 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 5 Jun 2024 02:09:29 -0500 Subject: [PATCH 30/43] test --- smartsim/entity/model.py | 6 +++--- smartsim/experiment.py | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index ecf96c8cb8..8b07ea2461 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -39,7 +39,7 @@ from .._core.utils.helpers import cat_arg_and_value, expand_exe_path from ..error import EntityExistsError, SSUnsupportedError from ..log import get_logger -# from ..settings.base import BatchSettings, RunSettings +from ..settings import BatchSettings, RunSettings from .dbobject import DBModel, DBScript from .entity import SmartSimEntity from .files import EntityFiles @@ -52,12 +52,12 @@ def __init__( self, name: str, exe: str, - # run_settings: RunSettings, + run_settings: RunSettings, params: t.Optional[t.Dict[str, str]] = None, exe_args: t.Optional[t.List[str]] = None, path: t.Optional[str] = getcwd(), params_as_args: t.Optional[t.List[str]] = None, - # batch_settings: t.Optional[BatchSettings] = None, + batch_settings: t.Optional[BatchSettings] = None, ): """Initialize a ``Model`` diff --git a/smartsim/experiment.py b/smartsim/experiment.py index dc00e581b1..1a4b687e79 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -48,11 +48,7 @@ ) from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer -# from .settings import Container, base, settings -# Mock imports -class Container: pass -class BatchSettings: pass -class RunSettings: pass +from .settings import Container, RunSettings, BatchSettings from .wlm import detect_launcher logger = get_logger(__name__) From 1b9e650d42bedf45cd4251dc8d8ede1a266cc601 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 5 Jun 2024 02:15:00 -0500 Subject: [PATCH 31/43] chnages --- smartsim/_core/launcher/dragon/dragonLauncher.py | 1 - smartsim/_core/launcher/launcher.py | 1 - smartsim/_core/launcher/step/alpsStep.py | 1 + smartsim/database/orchestrator.py | 1 - 4 files changed, 1 insertion(+), 3 deletions(-) diff --git a/smartsim/_core/launcher/dragon/dragonLauncher.py b/smartsim/_core/launcher/dragon/dragonLauncher.py index 23b2b3a046..17b47e3090 100644 --- a/smartsim/_core/launcher/dragon/dragonLauncher.py +++ b/smartsim/_core/launcher/dragon/dragonLauncher.py @@ -92,7 +92,6 @@ def cleanup(self) -> None: @property def supported_rs(self) -> t.Dict[t.Type[SettingsBase], t.Type[Step]]: # RunSettings types supported by this launcher - pass return { DragonRunSettings: DragonStep, SbatchSettings: DragonBatchStep, diff --git a/smartsim/_core/launcher/launcher.py b/smartsim/_core/launcher/launcher.py index 37e1447137..bbc9b59d62 100644 --- a/smartsim/_core/launcher/launcher.py +++ b/smartsim/_core/launcher/launcher.py @@ -31,7 +31,6 @@ from ...entity import SmartSimEntity from ...error import AllocationError, LauncherError, SSUnsupportedError from ...settings import SettingsBase - from .step import Step from .stepInfo import StepInfo, UnmanagedStepInfo from .stepMapping import StepMapping diff --git a/smartsim/_core/launcher/step/alpsStep.py b/smartsim/_core/launcher/step/alpsStep.py index 464f723a76..9b744498bd 100644 --- a/smartsim/_core/launcher/step/alpsStep.py +++ b/smartsim/_core/launcher/step/alpsStep.py @@ -37,6 +37,7 @@ logger = get_logger(__name__) + class AprunStep(Step): def __init__( self, entity: t.Union[Model, DBNode], run_settings: AprunSettings diff --git a/smartsim/database/orchestrator.py b/smartsim/database/orchestrator.py index 17544cdca8..46f666bd2e 100644 --- a/smartsim/database/orchestrator.py +++ b/smartsim/database/orchestrator.py @@ -64,7 +64,6 @@ ) from ..settings import BatchSettings, RunSettings - from ..settings import create_batch_settings, create_run_settings from ..wlm import detect_launcher From f79176ff476370197ed29df07dded84812ca03b1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 6 Jun 2024 14:23:46 -0500 Subject: [PATCH 32/43] fixes --- smartsim/_core/commands/command.py | 2 +- smartsim/_core/commands/commandList.py | 6 +- smartsim/_core/commands/launchCommands.py | 6 +- smartsim/settings/__init__.py | 1 + smartsim/settings/batchCommand.py | 6 +- smartsim/settings/batchSettings.py | 32 +++--- smartsim/settings/launchCommand.py | 18 ++-- smartsim/settings/launchSettings.py | 68 ++++++------ smartsim/settings/translators/__init__.py | 8 +- .../settings/translators/batch/__init.__.py | 12 +-- smartsim/settings/translators/batch/lsf.py | 9 +- smartsim/settings/translators/batch/pbs.py | 33 ++---- smartsim/settings/translators/batch/slurm.py | 4 +- ...tchArgTranslator.py => batchArgBuilder.py} | 3 +- .../settings/translators/launch/__init__.py | 32 +++--- smartsim/settings/translators/launch/alps.py | 5 +- .../settings/translators/launch/dragon.py | 4 +- smartsim/settings/translators/launch/local.py | 4 +- smartsim/settings/translators/launch/lsf.py | 5 +- smartsim/settings/translators/launch/mpi.py | 10 +- smartsim/settings/translators/launch/pals.py | 4 +- smartsim/settings/translators/launch/slurm.py | 4 +- ...chArgTranslator.py => launchArgBuilder.py} | 2 +- .../test_settings/test_alpsLauncher.py | 24 ++--- .../test_settings/test_batchSettings.py | 6 +- .../test_settings/test_dragonLauncher.py | 6 +- .../test_settings/test_launchSettings.py | 101 ++---------------- .../test_settings/test_localLauncher.py | 54 +++++----- .../test_settings/test_lsfLauncher.py | 14 +-- .../test_settings/test_lsfScheduler.py | 5 +- .../test_settings/test_mpiLauncher.py | 36 +++---- .../test_settings/test_palsLauncher.py | 8 +- .../test_settings/test_pbsScheduler.py | 13 +-- .../test_settings/test_slurmLauncher.py | 36 +++---- .../test_settings/test_slurmScheduler.py | 15 ++- 35 files changed, 247 insertions(+), 349 deletions(-) rename smartsim/settings/translators/{batchArgTranslator.py => batchArgBuilder.py} (97%) rename smartsim/settings/translators/{launchArgTranslator.py => launchArgBuilder.py} (98%) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 83b2bcb005..27ced30663 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -28,7 +28,7 @@ from ...settings.launchCommand import LauncherType import typing as t -class Command(MutableSequence): +class Command(MutableSequence[str]): """Basic container for command information """ def __init__(self, launcher: LauncherType, command:t.List[str]) -> None: diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 1fb94d19c2..0fb0d75764 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -28,12 +28,14 @@ from .command import Command import typing as t -class CommandList(MutableSequence): +class CommandList(MutableSequence[Command]): """Container for a Sequence of Command objects """ - def __init__(self, commands: t.Optional[t.Union[Command, t.List[Command]]]): + def __init__(self, commands: t.Union[Command, t.List[Command]]): """CommandList constructor """ + if isinstance(commands, Command): + commands = [commands] self._commands: t.List[Command] = list(commands) @property diff --git a/smartsim/_core/commands/launchCommands.py b/smartsim/_core/commands/launchCommands.py index daeada7534..a2dbcb6e0c 100644 --- a/smartsim/_core/commands/launchCommands.py +++ b/smartsim/_core/commands/launchCommands.py @@ -17,21 +17,21 @@ def __init__( self._postlaunch_commands = postlaunch_commands @property - def prelaunch_command_maps(self) -> CommandList: + def prelaunch_command(self) -> CommandList: """Get the prelaunch command list. Return a reference to the command list. """ return self._prelaunch_commands @property - def launch_command_maps(self) -> CommandList: + def launch_command(self) -> CommandList: """Get the launch command list. Return a reference to the command list. """ return self._launch_commands @property - def postlaunch_command_maps(self) -> CommandList: + def postlaunch_command(self) -> CommandList: """Get the postlaunch command list. Return a reference to the command list. """ diff --git a/smartsim/settings/__init__.py b/smartsim/settings/__init__.py index 2c52a3bedf..b8f40b03c3 100644 --- a/smartsim/settings/__init__.py +++ b/smartsim/settings/__init__.py @@ -34,6 +34,7 @@ "BatchSettings" ] +# TODO Mock imports for compiling tests class DragonRunSettings: pass class QsubBatchSettings: pass class SbatchSettings: pass diff --git a/smartsim/settings/batchCommand.py b/smartsim/settings/batchCommand.py index 8275c1728e..18aef8642f 100644 --- a/smartsim/settings/batchCommand.py +++ b/smartsim/settings/batchCommand.py @@ -30,6 +30,6 @@ class SchedulerType(Enum): """ Schedulers that are supported by SmartSim. """ - SlurmScheduler = "slurm" - PbsScheduler = "pbs" - LsfScheduler = "lsf" \ No newline at end of file + Slurm = "slurm" + Pbs = "pbs" + Lsf = "lsf" \ No newline at end of file diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 9954e87cb9..0372acd816 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -32,10 +32,10 @@ from .._core.utils.helpers import fmt_dict from .common import StringArgument from .batchCommand import SchedulerType -from .translators.batch.pbs import QsubBatchArgTranslator -from .translators.batch.slurm import SlurmBatchArgTranslator -from .translators.batch.lsf import BsubBatchArgTranslator -from .translators import BatchArgTranslator +from .translators.batch.pbs import QsubBatchArgBuilder +from .translators.batch.slurm import SlurmBatchArgBuilder +from .translators.batch.lsf import BsubBatchArgBuilder +from .translators import BatchArgBuilder from .baseSettings import BaseSettings logger = get_logger(__name__) @@ -51,7 +51,7 @@ def __init__( self._batch_scheduler = SchedulerType(batch_scheduler) except ValueError: raise ValueError(f"Invalid scheduler type: {batch_scheduler}") - self._arg_translator = self._get_arg_builder(scheduler_args) + self._arg_builder = self._get_arg_builder(scheduler_args) self.env_vars = env_vars or {} @property @@ -61,11 +61,11 @@ def batch_scheduler(self) -> str: return self._batch_scheduler.value @property - def scheduler_args(self) -> BatchArgTranslator: + def scheduler_args(self) -> BatchArgBuilder: """Return the batch argument translator. """ # Is a deep copy needed here? - return self._arg_translator + return self._arg_builder @property def env_vars(self) -> StringArgument: @@ -79,15 +79,15 @@ def env_vars(self, value: t.Mapping[str, str]) -> None: """ self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, scheduler_args) -> BatchArgTranslator: - """ Map the Scheduler to the BatchArgTranslator + def _get_arg_builder(self, scheduler_args) -> BatchArgBuilder: + """ Map the Scheduler to the BatchArgBuilder """ - if self._batch_scheduler == SchedulerType.SlurmScheduler: - return SlurmBatchArgTranslator(scheduler_args) - elif self._batch_scheduler == SchedulerType.LsfScheduler: - return BsubBatchArgTranslator(scheduler_args) - elif self._batch_scheduler == SchedulerType.PbsScheduler: - return QsubBatchArgTranslator(scheduler_args) + if self._batch_scheduler == SchedulerType.Slurm: + return SlurmBatchArgBuilder(scheduler_args) + elif self._batch_scheduler == SchedulerType.Lsf: + return BsubBatchArgBuilder(scheduler_args) + elif self._batch_scheduler == SchedulerType.Pbs: + return QsubBatchArgBuilder(scheduler_args) else: raise ValueError(f"Invalid scheduler type: {self._batch_scheduler}") @@ -96,7 +96,7 @@ def format_batch_args(self) -> t.List[str]: :return: batch arguments for Sbatch """ - return self._arg_translator.format_batch_args() + return self._arg_builder.format_batch_args() def __str__(self) -> str: # pragma: no-cover string = f"\nScheduler: {self.arg_translator.scheduler_str}" diff --git a/smartsim/settings/launchCommand.py b/smartsim/settings/launchCommand.py index b442bfad37..377bfec972 100644 --- a/smartsim/settings/launchCommand.py +++ b/smartsim/settings/launchCommand.py @@ -30,12 +30,12 @@ class LauncherType(Enum): """ Launchers that are supported by SmartSim. """ - DragonLauncher = "dragon" - SlurmLauncher = "slurm" - PalsLauncher = "pals" - AlpsLauncher = "alps" - LocalLauncher = "local" - MpiexecLauncher = "mpiexec" - MpirunLauncher = "mpirun" - OrterunLauncher = "orterun" - LsfLauncher = "lsf" \ No newline at end of file + Dragon = "dragon" + Slurm = "slurm" + Pals = "pals" + Alps = "alps" + Local = "local" + Mpiexec = "mpiexec" + Mpirun = "mpirun" + Orterun = "orterun" + Lsf = "lsf" \ No newline at end of file diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 7e39548a58..b52e4c3ad7 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -33,14 +33,14 @@ from .._core.utils.helpers import fmt_dict from .common import StringArgument from .launchCommand import LauncherType -from .translators.launch.alps import AprunArgTranslator -from .translators.launch.lsf import JsrunArgTranslator -from .translators.launch.mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator -from .translators.launch.pals import PalsMpiexecArgTranslator -from .translators.launch.slurm import SlurmArgTranslator -from .translators.launch.dragon import DragonArgTranslator -from .translators.launch.local import LocalArgTranslator -from .translators import LaunchArgTranslator +from .translators.launch.alps import AprunArgBuilder +from .translators.launch.lsf import JsrunArgBuilder +from .translators.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder +from .translators.launch.pals import PalsMpiexecArgBuilder +from .translators.launch.slurm import SlurmArgBuilder +from .translators.launch.dragon import DragonArgBuilder +from .translators.launch.local import LocalArgBuilder +from .translators import LaunchArgBuilder from .baseSettings import BaseSettings logger = get_logger(__name__) @@ -56,7 +56,7 @@ def __init__( self._launcher = LauncherType(launcher) except ValueError: raise ValueError(f"Invalid launcher type: {launcher}") - self._arg_translator = self._get_arg_builder(launch_args) + self._arg_builder = self._get_arg_builder(launch_args) self.env_vars = env_vars or {} @property @@ -66,11 +66,11 @@ def launcher(self) -> str: return self._launcher.value @property - def launch_args(self) -> LaunchArgTranslator: + def launch_args(self) -> LaunchArgBuilder: """Return the launch argument translator. """ # Is a deep copy needed here? - return self._arg_translator + return self._arg_builder @launch_args.setter def launch_args(self, args: t.Mapping[str, str]) -> None: @@ -92,27 +92,27 @@ def env_vars(self, value: t.Mapping[str, str]) -> None: """ self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, launch_args) -> LaunchArgTranslator: - """ Map the Launcher to the LaunchArgTranslator + def _get_arg_builder(self, launch_args) -> LaunchArgBuilder: + """ Map the Launcher to the LaunchArgBuilder """ - if self._launcher == LauncherType.SlurmLauncher: - return SlurmArgTranslator(launch_args) - elif self._launcher == LauncherType.MpiexecLauncher: - return MpiexecArgTranslator(launch_args) - elif self._launcher == LauncherType.MpirunLauncher: - return MpiArgTranslator(launch_args) - elif self._launcher == LauncherType.OrterunLauncher: - return OrteArgTranslator(launch_args) - elif self._launcher == LauncherType.AlpsLauncher: - return AprunArgTranslator(launch_args) - elif self._launcher == LauncherType.LsfLauncher: - return JsrunArgTranslator(launch_args) - elif self._launcher == LauncherType.PalsLauncher: - return PalsMpiexecArgTranslator(launch_args) - elif self._launcher == LauncherType.DragonLauncher: - return DragonArgTranslator(launch_args) - elif self._launcher == LauncherType.LocalLauncher: - return LocalArgTranslator(launch_args) + if self._launcher == LauncherType.Slurm: + return SlurmArgBuilder(launch_args) + elif self._launcher == LauncherType.Mpiexec: + return MpiexecArgBuilder(launch_args) + elif self._launcher == LauncherType.Mpirun: + return MpiArgBuilder(launch_args) + elif self._launcher == LauncherType.Orterun: + return OrteArgBuilder(launch_args) + elif self._launcher == LauncherType.Alps: + return AprunArgBuilder(launch_args) + elif self._launcher == LauncherType.Lsf: + return JsrunArgBuilder(launch_args) + elif self._launcher == LauncherType.Pals: + return PalsMpiexecArgBuilder(launch_args) + elif self._launcher == LauncherType.Dragon: + return DragonArgBuilder(launch_args) + elif self._launcher == LauncherType.Local: + return LocalArgBuilder(launch_args) else: raise ValueError(f"Invalid launcher type: {self._launcher}") @@ -140,7 +140,7 @@ def format_env_vars(self) -> t.Union[t.List[str],None]: """Build bash compatible environment variable string for Slurm :returns: the formatted string of environment variables """ - return self._arg_translator.format_env_vars(self.env_vars) + return self._arg_builder.format_env_vars(self.env_vars) def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: """Build environment variable string for Slurm @@ -149,7 +149,7 @@ def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: for more information on this, see the slurm documentation for srun :returns: the formatted string of environment variables """ - return self._arg_translator.format_comma_sep_env_vars(self.env_vars) + return self._arg_builder.format_comma_sep_env_vars(self.env_vars) def format_launch_args(self) -> t.Union[t.List[str],None]: """Return formatted launch arguments @@ -157,7 +157,7 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: literally with no formatting. :return: list run arguments for these settings """ - return self._arg_translator.format_launch_args() + return self._arg_builder.format_launch_args() def __str__(self) -> str: # pragma: no-cover string = f"\nLauncher: {self.launcher}" diff --git a/smartsim/settings/translators/__init__.py b/smartsim/settings/translators/__init__.py index 2831c0cb1c..be4bf2644d 100644 --- a/smartsim/settings/translators/__init__.py +++ b/smartsim/settings/translators/__init__.py @@ -24,10 +24,10 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .launchArgTranslator import LaunchArgTranslator -from .batchArgTranslator import BatchArgTranslator +from .launchArgBuilder import LaunchArgBuilder +from .batchArgBuilder import BatchArgBuilder __all__ = [ - "LaunchArgTranslator", - "BatchArgTranslator" + "LaunchArgBuilder", + "BatchArgBuilder" ] \ No newline at end of file diff --git a/smartsim/settings/translators/batch/__init.__.py b/smartsim/settings/translators/batch/__init.__.py index d3478e91fc..a232d8866b 100644 --- a/smartsim/settings/translators/batch/__init.__.py +++ b/smartsim/settings/translators/batch/__init.__.py @@ -24,12 +24,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .lsf import BsubBatchArgTranslator -from .pbs import QsubBatchArgTranslator -from .slurm import SlurmBatchArgTranslator +from .lsf import BsubBatchArgBuilder +from .pbs import QsubBatchArgBuilder +from .slurm import SlurmBatchArgBuilder __all__ = [ - "BsubBatchArgTranslator", - "QsubBatchArgTranslator", - "SlurmBatchArgTranslator", + "BsubBatchArgBuilder", + "QsubBatchArgBuilder", + "SlurmBatchArgBuilder", ] \ No newline at end of file diff --git a/smartsim/settings/translators/batch/lsf.py b/smartsim/settings/translators/batch/lsf.py index 7313855520..a6ed8dc525 100644 --- a/smartsim/settings/translators/batch/lsf.py +++ b/smartsim/settings/translators/batch/lsf.py @@ -27,14 +27,13 @@ from __future__ import annotations import typing as t -from ..batchArgTranslator import BatchArgTranslator -from ...common import IntegerArgument, StringArgument -from smartsim.log import get_logger -from ...batchCommand import SchedulerType +from ..batchArgBuilder import BatchArgBuilder +from ...common import StringArgument +from smartsim.log import get_logger logger = get_logger(__name__) -class BsubBatchArgTranslator(BatchArgTranslator): +class BsubBatchArgBuilder(BatchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/batch/pbs.py b/smartsim/settings/translators/batch/pbs.py index 05d5b6d86b..c8b9f9289d 100644 --- a/smartsim/settings/translators/batch/pbs.py +++ b/smartsim/settings/translators/batch/pbs.py @@ -28,14 +28,13 @@ from copy import deepcopy import typing as t -from ..batchArgTranslator import BatchArgTranslator +from ..batchArgBuilder import BatchArgBuilder from ....error import SSConfigError -from ...common import IntegerArgument, StringArgument -from smartsim.log import get_logger -from ...batchCommand import SchedulerType +from ...common import StringArgument +from smartsim.log import get_logger logger = get_logger(__name__) -class QsubBatchArgTranslator(BatchArgTranslator): +class QsubBatchArgBuilder(BatchArgBuilder): def __init__( self, @@ -125,7 +124,7 @@ def format_batch_args(self) -> t.List[str]: @staticmethod def _sanity_check_resources( - batch_args: t.Dict[str, t.Union[str,int,float,None]] + batch_args: t.Dict[str, str | None] ) -> None: """Check that only select or nodes was specified in resources @@ -143,25 +142,7 @@ def _sanity_check_resources( "'select' was set using 'set_resource'. Please only specify one." ) - # TODO ask the team if this is true below - # if has_select and not isinstance(has_select, int): - # raise TypeError("The value for 'select' must be an integer") - # if has_nodes and not isinstance(has_nodes, int): - # raise TypeError("The value for 'nodes' must be an integer") - - for key, value in batch_args.items(): - if not isinstance(key, str): - raise TypeError( - f"The type of {key} is {type(key)}. Only int and str " - "are allowed." - ) - if not isinstance(value, (str, int)): - raise TypeError( - f"The value associated with {key} is {type(value)}. Only int " - "and str are allowed." - ) - - def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.Tuple[t.List[str],t.Dict[str, t.Union[str,int,float,None]]]: + def _create_resource_list(self, batch_args: t.Dict[str, str | None]) -> t.Tuple[t.List[str],t.Dict[str, str | None]]: self._sanity_check_resources(batch_args) res = [] @@ -178,7 +159,7 @@ def _create_resource_list(self, batch_args: t.Dict[str, t.Union[str,int,float,No if ncpus := batch_arg_copy.pop("ppn", None): select_command += f":ncpus={ncpus}" if hosts := batch_arg_copy.pop("hostname", None): - hosts_list = ["=".join(str(hosts))] + hosts_list = ["=".join(("host", str(host))) for host in hosts.split(",")] select_command += f":{'+'.join(hosts_list)}" res += select_command.split() if walltime := batch_arg_copy.pop("walltime", None): diff --git a/smartsim/settings/translators/batch/slurm.py b/smartsim/settings/translators/batch/slurm.py index 2d33e4c76a..c26a7bf9db 100644 --- a/smartsim/settings/translators/batch/slurm.py +++ b/smartsim/settings/translators/batch/slurm.py @@ -28,13 +28,13 @@ import re import typing as t -from ..batchArgTranslator import BatchArgTranslator +from ..batchArgBuilder import BatchArgBuilder from ...common import StringArgument from smartsim.log import get_logger logger = get_logger(__name__) -class SlurmBatchArgTranslator(BatchArgTranslator): +class SlurmBatchArgBuilder(BatchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/batchArgTranslator.py b/smartsim/settings/translators/batchArgBuilder.py similarity index 97% rename from smartsim/settings/translators/batchArgTranslator.py rename to smartsim/settings/translators/batchArgBuilder.py index 8307d22121..723cc5421b 100644 --- a/smartsim/settings/translators/batchArgTranslator.py +++ b/smartsim/settings/translators/batchArgBuilder.py @@ -28,14 +28,13 @@ from abc import ABC, abstractmethod import typing as t -from ..common import IntegerArgument, StringArgument import copy from smartsim.log import get_logger logger = get_logger(__name__) -class BatchArgTranslator(ABC): +class BatchArgBuilder(ABC): """Abstract base class that defines all generic scheduler argument methods that are not supported. It is the responsibility of child classes for each launcher to translate diff --git a/smartsim/settings/translators/launch/__init__.py b/smartsim/settings/translators/launch/__init__.py index 3eba5e05db..7cc9844ba4 100644 --- a/smartsim/settings/translators/launch/__init__.py +++ b/smartsim/settings/translators/launch/__init__.py @@ -1,19 +1,19 @@ -from .alps import AprunArgTranslator -from .dragon import DragonArgTranslator -from .local import LocalArgTranslator -from .lsf import JsrunArgTranslator -from .mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator -from .pals import PalsMpiexecArgTranslator -from .slurm import SlurmArgTranslator +from .alps import AprunArgBuilder +from .dragon import DragonArgBuilder +from .local import LocalArgBuilder +from .lsf import JsrunArgBuilder +from .mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder +from .pals import PalsMpiexecArgBuilder +from .slurm import SlurmArgBuilder __all__ = [ - "AprunArgTranslator", - "DragonArgTranslator", - "LocalArgTranslator", - "JsrunArgTranslator", - "MpiArgTranslator", - "MpiexecArgTranslator", - "OrteArgTranslator", - "PalsMpiexecArgTranslator", - "SlurmArgTranslator", + "AprunArgBuilder", + "DragonArgBuilder", + "LocalArgBuilder", + "JsrunArgBuilder", + "MpiArgBuilder", + "MpiexecArgBuilder", + "OrteArgBuilder", + "PalsMpiexecArgBuilder", + "SlurmArgBuilder", ] \ No newline at end of file diff --git a/smartsim/settings/translators/launch/alps.py b/smartsim/settings/translators/launch/alps.py index 9ad35c58d0..68ed18db00 100644 --- a/smartsim/settings/translators/launch/alps.py +++ b/smartsim/settings/translators/launch/alps.py @@ -25,14 +25,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder import typing as t from ...common import set_check_input, StringArgument from smartsim.log import get_logger logger = get_logger(__name__) -class AprunArgTranslator(LaunchArgTranslator): +class AprunArgBuilder(LaunchArgBuilder): def __init__( self, @@ -184,7 +184,6 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: """ # args launcher uses args = [] - for opt, value in self._launch_args.items(): short_arg = bool(len(str(opt)) == 1) prefix = "-" if short_arg else "--" diff --git a/smartsim/settings/translators/launch/dragon.py b/smartsim/settings/translators/launch/dragon.py index abd0f63b3f..fd354f5b77 100644 --- a/smartsim/settings/translators/launch/dragon.py +++ b/smartsim/settings/translators/launch/dragon.py @@ -27,13 +27,13 @@ from __future__ import annotations import typing as t -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input from smartsim.log import get_logger logger = get_logger(__name__) -class DragonArgTranslator(LaunchArgTranslator): +class DragonArgBuilder(LaunchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/launch/local.py b/smartsim/settings/translators/launch/local.py index 6f0ce16304..432a18a6cb 100644 --- a/smartsim/settings/translators/launch/local.py +++ b/smartsim/settings/translators/launch/local.py @@ -27,13 +27,13 @@ from __future__ import annotations import typing as t -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder from smartsim.log import get_logger from ...common import StringArgument, set_check_input logger = get_logger(__name__) -class LocalArgTranslator(LaunchArgTranslator): +class LocalArgBuilder(LaunchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/launch/lsf.py b/smartsim/settings/translators/launch/lsf.py index e886868c68..3fa27044bf 100644 --- a/smartsim/settings/translators/launch/lsf.py +++ b/smartsim/settings/translators/launch/lsf.py @@ -27,14 +27,13 @@ from __future__ import annotations import typing as t -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input -from ...launchCommand import LauncherType from smartsim.log import get_logger logger = get_logger(__name__) -class JsrunArgTranslator(LaunchArgTranslator): +class JsrunArgBuilder(LaunchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/launch/mpi.py b/smartsim/settings/translators/launch/mpi.py index 8b56b075c5..05aec2506b 100644 --- a/smartsim/settings/translators/launch/mpi.py +++ b/smartsim/settings/translators/launch/mpi.py @@ -27,14 +27,14 @@ from __future__ import annotations import typing as t -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType from smartsim.log import get_logger logger = get_logger(__name__) -class _BaseMPIArgTranslator(LaunchArgTranslator): +class _BaseMPIArgBuilder(LaunchArgBuilder): def __init__( self, @@ -217,7 +217,7 @@ def set(self, key: str, value: str | None) -> None: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value -class MpiArgTranslator(_BaseMPIArgTranslator): +class MpiArgBuilder(_BaseMPIArgBuilder): def __init__( self, @@ -225,7 +225,7 @@ def __init__( ) -> None: super().__init__(launch_args) -class MpiexecArgTranslator(_BaseMPIArgTranslator): +class MpiexecArgBuilder(_BaseMPIArgBuilder): def __init__( self, @@ -233,7 +233,7 @@ def __init__( ) -> None: super().__init__(launch_args) -class OrteArgTranslator(_BaseMPIArgTranslator): +class OrteArgBuilder(_BaseMPIArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/launch/pals.py b/smartsim/settings/translators/launch/pals.py index a33eab6a81..e23629e936 100644 --- a/smartsim/settings/translators/launch/pals.py +++ b/smartsim/settings/translators/launch/pals.py @@ -27,13 +27,13 @@ from __future__ import annotations import typing as t -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input from smartsim.log import get_logger logger = get_logger(__name__) -class PalsMpiexecArgTranslator(LaunchArgTranslator): +class PalsMpiexecArgBuilder(LaunchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/launch/slurm.py b/smartsim/settings/translators/launch/slurm.py index 5de63b511c..314fdea20d 100644 --- a/smartsim/settings/translators/launch/slurm.py +++ b/smartsim/settings/translators/launch/slurm.py @@ -29,13 +29,13 @@ import typing as t import re import os -from ..launchArgTranslator import LaunchArgTranslator +from ..launchArgBuilder import LaunchArgBuilder from ...common import IntegerArgument, StringArgument, set_check_input from smartsim.log import get_logger logger = get_logger(__name__) -class SlurmArgTranslator(LaunchArgTranslator): +class SlurmArgBuilder(LaunchArgBuilder): def __init__( self, diff --git a/smartsim/settings/translators/launchArgTranslator.py b/smartsim/settings/translators/launchArgBuilder.py similarity index 98% rename from smartsim/settings/translators/launchArgTranslator.py rename to smartsim/settings/translators/launchArgBuilder.py index fcc5e3379a..5b5ddc9941 100644 --- a/smartsim/settings/translators/launchArgTranslator.py +++ b/smartsim/settings/translators/launchArgBuilder.py @@ -33,7 +33,7 @@ logger = get_logger(__name__) -class LaunchArgTranslator(ABC): +class LaunchArgBuilder(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index c47571b231..fd4f05b30f 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,6 +1,6 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.alps import AprunArgTranslator +from smartsim.settings.translators.launch.alps import AprunArgBuilder from smartsim.settings.launchCommand import LauncherType @pytest.mark.parametrize( @@ -23,22 +23,22 @@ ], ) def test_alps_class_methods(function, value, flag, result): - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert isinstance(alpsLauncher._arg_translator,AprunArgTranslator) + alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) + assert isinstance(alpsLauncher._arg_builder,AprunArgBuilder) getattr(alpsLauncher.launch_args, function)(*value) assert alpsLauncher.launch_args._launch_args[flag] == result def test_set_verbose_launch(): - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert isinstance(alpsLauncher._arg_translator,AprunArgTranslator) + alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) + assert isinstance(alpsLauncher._arg_builder,AprunArgBuilder) alpsLauncher.launch_args.set_verbose_launch(True) assert alpsLauncher.launch_args._launch_args == {'debug': "7"} alpsLauncher.launch_args.set_verbose_launch(False) assert alpsLauncher.launch_args._launch_args == {} def test_set_quiet_launch(): - aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) - assert isinstance(aprunLauncher._arg_translator,AprunArgTranslator) + aprunLauncher = LaunchSettings(launcher=LauncherType.Alps) + assert isinstance(aprunLauncher._arg_builder,AprunArgBuilder) aprunLauncher.launch_args.set_quiet_launch(True) assert aprunLauncher.launch_args._launch_args == {'quiet': None} aprunLauncher.launch_args.set_quiet_launch(False) @@ -46,15 +46,15 @@ def test_set_quiet_launch(): def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} - aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher, env_vars=env_vars) - assert isinstance(aprunLauncher._arg_translator,AprunArgTranslator) + aprunLauncher = LaunchSettings(launcher=LauncherType.Alps, env_vars=env_vars) + assert isinstance(aprunLauncher._arg_builder,AprunArgBuilder) aprunLauncher.update_env({"OMP_NUM_THREADS": "10"}) formatted = aprunLauncher.format_env_vars() result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] assert formatted == result def test_aprun_settings(): - aprunLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + aprunLauncher = LaunchSettings(launcher=LauncherType.Alps) aprunLauncher.launch_args.set_cpus_per_task(2) aprunLauncher.launch_args.set_tasks(100) aprunLauncher.launch_args.set_tasks_per_node(20) @@ -64,7 +64,7 @@ def test_aprun_settings(): def test_invalid_hostlist_format(): """Test invalid hostlist formats""" - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) with pytest.raises(TypeError): alpsLauncher.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): @@ -74,7 +74,7 @@ def test_invalid_hostlist_format(): def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" - alpsLauncher = LaunchSettings(launcher=LauncherType.AlpsLauncher) + alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) with pytest.raises(TypeError): alpsLauncher.launch_args.set_excluded_hosts(["test",5]) with pytest.raises(TypeError): diff --git a/tests/temp_tests/test_settings/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py index ec06e77114..323268a2cb 100644 --- a/tests/temp_tests/test_settings/test_batchSettings.py +++ b/tests/temp_tests/test_settings/test_batchSettings.py @@ -5,9 +5,9 @@ @pytest.mark.parametrize( "scheduler_enum", [ - pytest.param(SchedulerType.SlurmScheduler, id="slurm"), - pytest.param(SchedulerType.PbsScheduler, id="dragon"), - pytest.param(SchedulerType.LsfScheduler, id="lsf"), + pytest.param(SchedulerType.Slurm, id="slurm"), + pytest.param(SchedulerType.Pbs, id="dragon"), + pytest.param(SchedulerType.Lsf, id="lsf"), ], ) def test_create_scheduler_settings(scheduler_enum): diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index bb95350c01..6fbadbf85e 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,5 +1,5 @@ from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.dragon import DragonArgTranslator +from smartsim.settings.translators.launch.dragon import DragonArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -12,7 +12,7 @@ ], ) def test_dragon_class_methods(function, value, flag, result): - dragonLauncher = LaunchSettings(launcher=LauncherType.DragonLauncher) - assert isinstance(dragonLauncher._arg_translator,DragonArgTranslator) + dragonLauncher = LaunchSettings(launcher=LauncherType.Dragon) + assert isinstance(dragonLauncher._arg_builder,DragonArgBuilder) getattr(dragonLauncher.launch_args, function)(*value) assert dragonLauncher.launch_args._launch_args[flag] == result \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py index dbfe0bfbfb..f9e569bd0f 100644 --- a/tests/temp_tests/test_settings/test_launchSettings.py +++ b/tests/temp_tests/test_settings/test_launchSettings.py @@ -6,15 +6,15 @@ @pytest.mark.parametrize( "launch_enum", [ - pytest.param(LauncherType.SlurmLauncher,id="slurm"), - pytest.param(LauncherType.DragonLauncher,id="dragon"), - pytest.param(LauncherType.PalsLauncher,id="pals"), - pytest.param(LauncherType.AlpsLauncher,id="alps"), - pytest.param(LauncherType.LocalLauncher,id="local"), - pytest.param(LauncherType.MpiexecLauncher,id="mpiexec"), - pytest.param(LauncherType.MpirunLauncher,id="mpirun"), - pytest.param(LauncherType.OrterunLauncher,id="orterun"), - pytest.param(LauncherType.LsfLauncher,id="lsf"), + pytest.param(LauncherType.Slurm,id="slurm"), + pytest.param(LauncherType.Dragon,id="dragon"), + pytest.param(LauncherType.Pals,id="pals"), + pytest.param(LauncherType.Alps,id="alps"), + pytest.param(LauncherType.Local,id="local"), + pytest.param(LauncherType.Mpiexec,id="mpiexec"), + pytest.param(LauncherType.Mpirun,id="mpirun"), + pytest.param(LauncherType.Orterun,id="orterun"), + pytest.param(LauncherType.Lsf,id="lsf"), ], ) def test_create_launch_settings(launch_enum): @@ -59,85 +59,4 @@ def test_update_env_vars_errors(): # Make sure the first key and value do not assign # and that the function is atomic ls.update_env({"test":"test","test":1}) - assert ls.env_vars == {"ENV":"VAR"} - -# TODO need to test launch_args -# def test_set_launch_args(): -# ls = LaunchSettings(launcher="local", launch_args = {"init":"arg"}) -# assert ls.launch_args == {"init":"arg"} -# ls.launch_args = {"launch":"arg"} -# assert ls.launch_args == {"launch":"arg"} - -# @pytest.mark.parametrize( -# "launcher,key", -# [ -# pytest.param(LauncherType.SlurmLauncher, ("chdir",), id="slurm-chdir"), -# pytest.param(LauncherType.SlurmLauncher, ("D",), id="slurm-D"), -# pytest.param(LauncherType.LsfLauncher, ("chdir",), id="lsf-chdir"), -# pytest.param(LauncherType.LsfLauncher, ("h",), id="lsf-h"), -# pytest.param(LauncherType.MpiexecLauncher, ("wd",), id="mpiexec-wd"), -# pytest.param(LauncherType.OrterunLauncher, ("wd",), id="orte-wd"), -# pytest.param(LauncherType.MpirunLauncher, ("wd",), id="mpi-wd"), -# pytest.param(LauncherType.MpiexecLauncher, ("wdir",), id="mpiexec-wdir"), -# pytest.param(LauncherType.OrterunLauncher, ("wdir",), id="orte-wdir"), -# pytest.param(LauncherType.MpirunLauncher, ("wdir",), id="mpi-wdir"), -# ], -# ) -# def test_prevent_set_reserved_launch_args(caplog, launcher, key): -# """Test methods not implemented throw warnings""" -# from smartsim.settings.launchSettings import logger - -# prev_prop = logger.propagate -# logger.propagate = True - -# with caplog.at_level(logging.WARNING): -# caplog.clear() -# launchSettings = LaunchSettings(launcher=launcher) -# try: -# getattr(launchSettings, "set")(*key, None) -# finally: -# logger.propagate = prev_prop - -# for rec in caplog.records: -# if ( -# logging.WARNING <= rec.levelno < logging.ERROR -# and "Could not set argument" in rec.msg -# ): -# break -# else: -# pytest.fail( -# ( -# f"No message stating method `{key}` is not " -# "implemented at `warning` level" -# ) -# ) - -# def test_log_overwrite_set_warning_message(caplog): -# """Test methods not implemented throw warnings""" -# from smartsim.settings.launchSettings import logger - -# prev_prop = logger.propagate -# logger.propagate = True - -# with caplog.at_level(logging.WARNING): -# caplog.clear() -# launchSettings = LaunchSettings(launcher=LauncherType.LocalLauncher) -# launchSettings.set("test", None) -# try: -# getattr(launchSettings, "set")("test", "overwritting") -# finally: -# logger.propagate = prev_prop - -# for rec in caplog.records: -# if ( -# logging.WARNING <= rec.levelno < logging.ERROR -# and "Overwritting argument" in rec.msg -# ): -# break -# else: -# pytest.fail( -# ( -# f"No message stating method `test` will be " -# "overwritten at `warning` level" -# ) -# ) \ No newline at end of file + assert ls.env_vars == {"ENV":"VAR"} \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 68b86d2630..badb2384aa 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,33 +1,33 @@ from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.local import LocalArgTranslator +from smartsim.settings.translators.launch.local import LocalArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest import logging # TODO complete after launch args retrieval -# def test_launch_args_input_mutation(): -# # Tests that the run args passed in are not modified after initialization -# key0, key1, key2 = "arg0", "arg1", "arg2" -# val0, val1, val2 = "val0", "val1", "val2" +def test_launch_args_input_mutation(): + # Tests that the run args passed in are not modified after initialization + key0, key1, key2 = "arg0", "arg1", "arg2" + val0, val1, val2 = "val0", "val1", "val2" -# default_launcher_args = { -# key0: val0, -# key1: val1, -# key2: val2, -# } -# localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launch_args=default_launcher_args) + default_launcher_args = { + key0: val0, + key1: val1, + key2: val2, + } + localLauncher = LaunchSettings(launcher=LauncherType.Local, launch_args=default_launcher_args) -# # Confirm initial values are set -# assert localLauncher.launcher_args[key0] == val0 -# assert localLauncher.launcher_args[key1] == val1 -# assert localLauncher.launcher_args[key2] == val2 + # Confirm initial values are set + assert localLauncher.launch_args._launch_args[key0] == val0 + assert localLauncher.launch_args._launch_args[key1] == val1 + assert localLauncher.launch_args._launch_args[key2] == val2 -# # Update our common run arguments -# val2_upd = f"not-{val2}" -# default_launcher_args[key2] = val2_upd + # Update our common run arguments + val2_upd = f"not-{val2}" + default_launcher_args[key2] = val2_upd -# # Confirm previously created run settings are not changed -# assert localLauncher.launcher_args[key2] == val2 + # Confirm previously created run settings are not changed + assert localLauncher.launch_args._launch_args[key2] == val2 @pytest.mark.parametrize( "env_vars", @@ -40,15 +40,15 @@ ) def test_update_env(env_vars): """Ensure non-initialized env vars update correctly""" - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher) + localLauncher = LaunchSettings(launcher=LauncherType.Local) localLauncher.update_env(env_vars) assert len(localLauncher.env_vars) == len(env_vars.keys()) def test_format_launch_args(): - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, launch_args={"-np": 2}) + localLauncher = LaunchSettings(launcher=LauncherType.Local, launch_args={"-np": 2}) launch_args = localLauncher.format_launch_args() - assert launch_args == ["np", "2"] + assert launch_args == ["-np", "2"] @pytest.mark.parametrize( "env_vars", @@ -63,7 +63,7 @@ def test_update_env_null_valued(env_vars): orig_env = {} with pytest.raises(TypeError) as ex: - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=orig_env) + localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=orig_env) localLauncher.update_env(env_vars) @pytest.mark.parametrize( @@ -78,7 +78,7 @@ def test_update_env_null_valued(env_vars): def test_update_env_initialized(env_vars): """Ensure update of initialized env vars does not overwrite""" orig_env = {"key": "value"} - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=orig_env) + localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=orig_env) localLauncher.update_env(env_vars) combined_keys = {k for k in env_vars.keys()} @@ -94,6 +94,6 @@ def test_format_env_vars(): "C": "", "D": "12", } - localLauncher = LaunchSettings(launcher=LauncherType.LocalLauncher, env_vars=env_vars) - assert isinstance(localLauncher._arg_translator, LocalArgTranslator) + localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=env_vars) + assert isinstance(localLauncher._arg_builder, LocalArgBuilder) assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index 833c6d62cc..b679c9845f 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,5 +1,5 @@ from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.lsf import JsrunArgTranslator +from smartsim.settings.translators.launch.lsf import JsrunArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest @@ -11,15 +11,15 @@ ], ) def test_lsf_class_methods(function, value, flag, result): - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher) - assert isinstance(lsfLauncher._arg_translator,JsrunArgTranslator) + lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf) + assert isinstance(lsfLauncher._arg_builder,JsrunArgBuilder) getattr(lsfLauncher.launch_args, function)(*value) assert lsfLauncher.launch_args._launch_args[flag] == result def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, env_vars=env_vars) - assert isinstance(lsfLauncher._arg_translator,JsrunArgTranslator) + lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf, env_vars=env_vars) + assert isinstance(lsfLauncher._arg_builder,JsrunArgBuilder) formatted = lsfLauncher.format_env_vars() assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] @@ -32,8 +32,8 @@ def test_launch_args(): "nrs": 10, "np": 100, } - lsfLauncher = LaunchSettings(launcher=LauncherType.LsfLauncher, launch_args=launch_args) - assert isinstance(lsfLauncher._arg_translator,JsrunArgTranslator) + lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf, launch_args=launch_args) + assert isinstance(lsfLauncher._arg_builder,JsrunArgBuilder) formatted = lsfLauncher.format_launch_args() result = [ "--latency_priority=gpu-gpu", diff --git a/tests/temp_tests/test_settings/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py index 25860176b5..baf3b2e74d 100644 --- a/tests/temp_tests/test_settings/test_lsfScheduler.py +++ b/tests/temp_tests/test_settings/test_lsfScheduler.py @@ -1,6 +1,5 @@ from smartsim.settings import BatchSettings import pytest -import logging from smartsim.settings.batchCommand import SchedulerType @pytest.mark.parametrize( @@ -18,13 +17,13 @@ ], ) def test_update_env_initialized(function, value, flag, result): - lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler) + lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.Lsf) getattr(lsfScheduler.scheduler_args, function)(*value) assert lsfScheduler.scheduler_args._scheduler_args[flag] == result def test_create_bsub(): batch_args = {"core_isolation": None} - lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.LsfScheduler, scheduler_args=batch_args) + lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.Lsf, scheduler_args=batch_args) lsfScheduler.scheduler_args.set_nodes(1) lsfScheduler.scheduler_args.set_walltime("10:10:10") lsfScheduler.scheduler_args.set_queue("default") diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index b0b9d0ca6e..5c6ea12ddb 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -1,5 +1,5 @@ from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.mpi import MpiArgTranslator, MpiexecArgTranslator, OrteArgTranslator +from smartsim.settings.translators.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder import pytest import logging import itertools @@ -23,22 +23,22 @@ pytest.param(l, "set_hostlist", (["host_A","host_B"],),"host_A,host_B","host",id="set_hostlist_list[str]"), pytest.param(l, "set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","hostfile",id="set_hostlist_from_file"), ) - for l in ([LauncherType.MpirunLauncher, MpiArgTranslator], [LauncherType.MpiexecLauncher, MpiexecArgTranslator], [LauncherType.OrterunLauncher, OrteArgTranslator]) + for l in ([LauncherType.Mpirun, MpiArgBuilder], [LauncherType.Mpiexec, MpiexecArgBuilder], [LauncherType.Orterun, OrteArgBuilder]) )) ], ) def test_mpi_class_methods(l,function, value, flag, result): mpiSettings = LaunchSettings(launcher=l[0]) - assert isinstance(mpiSettings._arg_translator,l[1]) + assert isinstance(mpiSettings._arg_builder,l[1]) getattr(mpiSettings.launch_args, function)(*value) assert mpiSettings.launch_args._launch_args[flag] == result @pytest.mark.parametrize( "launcher", [ - pytest.param(LauncherType.MpirunLauncher, id="format_env_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="format_env_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="format_env_orterun"), + pytest.param(LauncherType.Mpirun, id="format_env_mpirun"), + pytest.param(LauncherType.Mpiexec, id="format_env_mpiexec"), + pytest.param(LauncherType.Orterun, id="format_env_orterun"), ], ) def test_format_env_vars(launcher): @@ -56,9 +56,9 @@ def test_format_env_vars(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param(LauncherType.MpirunLauncher, id="format_launcher_args_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="format_launcher_args_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="format_launcher_args_orterun"), + pytest.param(LauncherType.Mpirun, id="format_launcher_args_mpirun"), + pytest.param(LauncherType.Mpiexec, id="format_launcher_args_mpiexec"), + pytest.param(LauncherType.Orterun, id="format_launcher_args_orterun"), ], ) def test_format_launcher_args(launcher): @@ -73,9 +73,9 @@ def test_format_launcher_args(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param(LauncherType.MpirunLauncher, id="set_verbose_launch_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="set_verbose_launch_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="set_verbose_launch_orterun"), + pytest.param(LauncherType.Mpirun, id="set_verbose_launch_mpirun"), + pytest.param(LauncherType.Mpiexec, id="set_verbose_launch_mpiexec"), + pytest.param(LauncherType.Orterun, id="set_verbose_launch_orterun"), ], ) def test_set_verbose_launch(launcher): @@ -88,9 +88,9 @@ def test_set_verbose_launch(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param(LauncherType.MpirunLauncher, id="set_quiet_launch_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="set_quiet_launch_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="set_quiet_launch_orterun"), + pytest.param(LauncherType.Mpirun, id="set_quiet_launch_mpirun"), + pytest.param(LauncherType.Mpiexec, id="set_quiet_launch_mpiexec"), + pytest.param(LauncherType.Orterun, id="set_quiet_launch_orterun"), ], ) def test_set_quiet_launch(launcher): @@ -103,9 +103,9 @@ def test_set_quiet_launch(launcher): @pytest.mark.parametrize( "launcher", [ - pytest.param(LauncherType.MpirunLauncher, id="invalid_hostlist_mpirun"), - pytest.param(LauncherType.MpiexecLauncher, id="invalid_hostlist_mpiexec"), - pytest.param(LauncherType.OrterunLauncher, id="invalid_hostlist_orterun"), + pytest.param(LauncherType.Mpirun, id="invalid_hostlist_mpirun"), + pytest.param(LauncherType.Mpiexec, id="invalid_hostlist_mpiexec"), + pytest.param(LauncherType.Orterun, id="invalid_hostlist_orterun"), ], ) def test_invalid_hostlist_format(launcher): diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index f8b4908f5e..b64fdbe54b 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -1,5 +1,5 @@ from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.pals import PalsMpiexecArgTranslator +from smartsim.settings.translators.launch.pals import PalsMpiexecArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest @@ -15,21 +15,21 @@ ], ) def test_pals_class_methods(function, value, flag, result): - palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) + palsLauncher = LaunchSettings(launcher=LauncherType.Pals) getattr(palsLauncher.launch_args, function)(*value) assert palsLauncher.launch_args._launch_args[flag] == result assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] def test_format_env_vars(): env_vars = {"FOO_VERSION": "3.14", "PATH": None, "LD_LIBRARY_PATH": None} - palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher, env_vars=env_vars) + palsLauncher = LaunchSettings(launcher=LauncherType.Pals, env_vars=env_vars) formatted = " ".join(palsLauncher.format_env_vars()) expected = "--env FOO_VERSION=3.14 --envlist PATH,LD_LIBRARY_PATH" assert formatted == expected def test_invalid_hostlist_format(): """Test invalid hostlist formats""" - palsLauncher = LaunchSettings(launcher=LauncherType.PalsLauncher) + palsLauncher = LaunchSettings(launcher=LauncherType.Pals) with pytest.raises(TypeError): palsLauncher.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index 0c7e33522a..34bcb38ec8 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -1,5 +1,5 @@ from smartsim.settings import BatchSettings -from smartsim.settings.translators.batch.pbs import QsubBatchArgTranslator +from smartsim.settings.translators.batch.pbs import QsubBatchArgBuilder from smartsim.settings.batchCommand import SchedulerType import pytest @@ -15,21 +15,22 @@ pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hostname",id="set_hostlist_list[str]"), ], ) -def test_update_env_initialized(function, value, flag, result): - pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) +def test_create_pbs_batch(function, value, flag, result): + pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.Pbs) getattr(pbsScheduler.scheduler_args, function)(*value) assert pbsScheduler.scheduler_args._scheduler_args[flag] == result -def test_create_pbs_batch(): - pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.PbsScheduler) +def test_format_pbs_batch_args(): + pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.Pbs) pbsScheduler.scheduler_args.set_nodes(1) pbsScheduler.scheduler_args.set_walltime("10:00:00") pbsScheduler.scheduler_args.set_queue("default") pbsScheduler.scheduler_args.set_account("myproject") pbsScheduler.scheduler_args.set_ncpus(10) + pbsScheduler.scheduler_args.set_hostlist(['host_a', 'host_b', 'host_c']) args = pbsScheduler.format_batch_args() assert args == [ - "-l", "nodes=1:ncpus=10", + "-l", "nodes=1:ncpus=10:host=host_a+host=host_b+host=host_c", "-l", "walltime=10:00:00", "-q", "default", "-A", "myproject", diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index abc1f2798f..3ccf72388e 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,5 +1,5 @@ from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.slurm import SlurmArgTranslator +from smartsim.settings.translators.launch.slurm import SlurmArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest import logging @@ -25,20 +25,20 @@ ], ) def test_slurm_class_methods(function, value, flag, result): - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) - assert isinstance(slurmLauncher.launch_args,SlurmArgTranslator) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) + assert isinstance(slurmLauncher.launch_args,SlurmArgBuilder) getattr(slurmLauncher.launch_args, function)(*value) assert slurmLauncher.launch_args._launch_args[flag] == result def test_set_verbose_launch(): - ls = LaunchSettings(launcher=LauncherType.SlurmLauncher) + ls = LaunchSettings(launcher=LauncherType.Slurm) ls.launch_args.set_verbose_launch(True) assert ls.launch_args._launch_args == {'verbose': None} ls.launch_args.set_verbose_launch(False) assert ls.launch_args._launch_args == {} def test_set_quiet_launch(): - ls = LaunchSettings(launcher=LauncherType.SlurmLauncher) + ls = LaunchSettings(launcher=LauncherType.Slurm) ls.launch_args.set_quiet_launch(True) assert ls.launch_args._launch_args == {'quiet': None} ls.launch_args.set_quiet_launch(False) @@ -51,7 +51,7 @@ def test_format_env_vars(): "LOGGING": "verbose", "SSKEYIN": "name_0,name_1", } - ls = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) + ls = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) ls_format = ls.format_env_vars() assert "OMP_NUM_THREADS=20" in ls_format assert "LOGGING=verbose" in ls_format @@ -59,7 +59,7 @@ def test_format_env_vars(): def test_catch_existing_env_var(caplog, monkeypatch): slurmSettings = LaunchSettings( - launcher=LauncherType.SlurmLauncher, + launcher=LauncherType.Slurm, env_vars={ "SMARTSIM_TEST_VAR": "B", }, @@ -80,7 +80,7 @@ def test_catch_existing_env_var(caplog, monkeypatch): caplog.clear() env_vars = {"SMARTSIM_TEST_VAR": "B", "SMARTSIM_TEST_CSVAR": "C,D"} - settings = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) + settings = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) settings.format_comma_sep_env_vars() for record in caplog.records: @@ -90,7 +90,7 @@ def test_catch_existing_env_var(caplog, monkeypatch): def test_format_comma_sep_env_vars(): """Test format_comma_sep_env_vars runs correctly""" env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, env_vars=env_vars) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) formatted, comma_separated_formatted = slurmLauncher.format_comma_sep_env_vars() assert "OMP_NUM_THREADS" in formatted assert "LOGGING" in formatted @@ -100,7 +100,7 @@ def test_format_comma_sep_env_vars(): def test_slurmSettings_settings(): """Test format_launch_args runs correctly""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) slurmLauncher.launch_args.set_nodes(5) slurmLauncher.launch_args.set_cpus_per_task(2) slurmLauncher.launch_args.set_tasks(100) @@ -118,7 +118,7 @@ def test_slurmSettings_launch_args(): "nodes": 10, "ntasks": 100, } - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher, launch_args=launch_args) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm, launch_args=launch_args) formatted = slurmLauncher.format_launch_args() result = [ "--account=A3123", @@ -132,7 +132,7 @@ def test_slurmSettings_launch_args(): def test_invalid_hostlist_format(): """Test invalid hostlist formats""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(TypeError): slurmLauncher.launch_args.set_hostlist(["test",5]) with pytest.raises(TypeError): @@ -142,7 +142,7 @@ def test_invalid_hostlist_format(): def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(TypeError): slurmLauncher.launch_args.set_excluded_hosts(["test",5]) with pytest.raises(TypeError): @@ -152,7 +152,7 @@ def test_invalid_exclude_hostlist_format(): def test_invalid_node_feature_format(): """Test invalid node feature formats""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(TypeError): slurmLauncher.launch_args.set_node_feature(["test",5]) with pytest.raises(TypeError): @@ -162,7 +162,7 @@ def test_invalid_node_feature_format(): def test_invalid_walltime_format(): """Test invalid walltime formats""" - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(ValueError): slurmLauncher.launch_args.set_walltime("11:11") with pytest.raises(ValueError): @@ -175,10 +175,10 @@ def test_invalid_walltime_format(): def test_set_het_groups(monkeypatch): """Test ability to set one or more het groups to run setting""" monkeypatch.setenv("SLURM_HET_SIZE", "4") - slurmLauncher = LaunchSettings(launcher=LauncherType.SlurmLauncher) + slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) slurmLauncher.launch_args.set_het_group([1]) - assert slurmLauncher._arg_translator._launch_args["het-group"] == "1" + assert slurmLauncher._arg_builder._launch_args["het-group"] == "1" slurmLauncher.launch_args.set_het_group([3, 2]) - assert slurmLauncher._arg_translator._launch_args["het-group"] == "3,2" + assert slurmLauncher._arg_builder._launch_args["het-group"] == "3,2" with pytest.raises(ValueError): slurmLauncher.launch_args.set_het_group([4]) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index 92a0d530f3..9cd6d4725d 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -1,5 +1,5 @@ from smartsim.settings import BatchSettings -from smartsim.settings.translators.batch.slurm import SlurmBatchArgTranslator +from smartsim.settings.translators.batch.slurm import SlurmBatchArgBuilder from smartsim.settings.batchCommand import SchedulerType import pytest @@ -17,15 +17,14 @@ ], ) def test_sbatch_class_methods(function, value, flag, result): - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm) getattr(slurmScheduler.scheduler_args, function)(*value) assert slurmScheduler.scheduler_args._scheduler_args[flag] == result def test_create_sbatch(): batch_args = {"exclusive": None, "oversubscribe": None} - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args=batch_args) - assert isinstance(slurmScheduler._arg_translator, SlurmBatchArgTranslator) - #assert slurmScheduler.batch_args["partition"] == "default" + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm, scheduler_args=batch_args) + assert isinstance(slurmScheduler._arg_builder, SlurmBatchArgBuilder) args = slurmScheduler.format_batch_args() assert args == [ "--exclusive", @@ -42,7 +41,7 @@ def test_launch_args_input_mutation(): key1: val1, key2: val2, } - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler, scheduler_args=default_scheduler_args) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm, scheduler_args=default_scheduler_args) # Confirm initial values are set assert slurmScheduler.scheduler_args._scheduler_args[key0] == val0 @@ -58,14 +57,14 @@ def test_launch_args_input_mutation(): def test_sbatch_settings(): scheduler_args = {"nodes": 1, "time": "10:00:00", "account": "A3123"} - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler,scheduler_args=scheduler_args) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm,scheduler_args=scheduler_args) formatted = slurmScheduler.format_batch_args() result = ["--nodes=1", "--time=10:00:00", "--account=A3123"] assert formatted == result def test_sbatch_manual(): - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.SlurmScheduler) + slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm) slurmScheduler.scheduler_args.set_nodes(5) slurmScheduler.scheduler_args.set_account("A3531") slurmScheduler.scheduler_args.set_walltime("10:00:00") From d756c78d5e88c3ad1cfebd8b32af908d6749ed2f Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 6 Jun 2024 15:19:44 -0500 Subject: [PATCH 33/43] mypy errors --- smartsim/_core/commands/launchCommands.py | 6 +-- smartsim/settings/batchSettings.py | 18 ++++++--- smartsim/settings/common.py | 5 ++- smartsim/settings/launchSettings.py | 18 ++++----- smartsim/settings/translators/batch/lsf.py | 10 ++++- smartsim/settings/translators/batch/pbs.py | 8 +++- smartsim/settings/translators/batch/slurm.py | 8 +++- .../settings/translators/batchArgBuilder.py | 10 ++++- smartsim/settings/translators/launch/alps.py | 12 ++++-- .../settings/translators/launch/dragon.py | 12 ++++-- smartsim/settings/translators/launch/local.py | 12 ++++-- smartsim/settings/translators/launch/lsf.py | 12 ++++-- smartsim/settings/translators/launch/mpi.py | 29 +++++++++++---- smartsim/settings/translators/launch/pals.py | 12 ++++-- smartsim/settings/translators/launch/slurm.py | 12 ++++-- .../settings/translators/launchArgBuilder.py | 37 ++++++++++++++++++- .../test_settings/test_dragonLauncher.py | 1 - .../test_settings/test_localLauncher.py | 1 - .../test_settings/test_mpiLauncher.py | 1 - .../test_settings/test_palsLauncher.py | 1 + .../test_settings/test_pbsScheduler.py | 1 + .../test_settings/test_slurmLauncher.py | 1 - 22 files changed, 171 insertions(+), 56 deletions(-) diff --git a/smartsim/_core/commands/launchCommands.py b/smartsim/_core/commands/launchCommands.py index a2dbcb6e0c..96d1203e1b 100644 --- a/smartsim/_core/commands/launchCommands.py +++ b/smartsim/_core/commands/launchCommands.py @@ -39,12 +39,12 @@ def postlaunch_command(self) -> CommandList: def __str__(self) -> str: # pragma: no cover string = "\n\nPrelaunch Command List:\n" - for _, pre_cmd in enumerate(self.prelaunch_command_maps): + for _, pre_cmd in enumerate(self.prelaunch_command): string += f"{pre_cmd}\n" string += "\n\nLaunch Command List:\n" - for _, launch_cmd in enumerate(self.launch_command_maps): + for _, launch_cmd in enumerate(self.launch_command): string += f"{launch_cmd}\n" string += "\n\nPostlaunch Command List:\n" - for _, post_cmd in enumerate(self.postlaunch_command_maps): + for _, post_cmd in enumerate(self.postlaunch_command): string += f"{post_cmd}\n" return string \ No newline at end of file diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 0372acd816..6a2bdd8fe2 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -44,8 +44,8 @@ class BatchSettings(BaseSettings): def __init__( self, batch_scheduler: t.Union[SchedulerType, str], - scheduler_args: t.Optional[t.Dict[str, t.Union[str,int,float,None]]] = None, - env_vars: t.Optional[StringArgument] = None, + scheduler_args: t.Dict[str, t.Union[str, None]] | None = None, + env_vars: StringArgument | None = None, ) -> None: try: self._batch_scheduler = SchedulerType(batch_scheduler) @@ -54,6 +54,12 @@ def __init__( self._arg_builder = self._get_arg_builder(scheduler_args) self.env_vars = env_vars or {} + @property + def scheduler(self) -> str: + """Return the launcher name. + """ + return self._batch_scheduler.value + @property def batch_scheduler(self) -> str: """Return the scheduler name. @@ -74,12 +80,12 @@ def env_vars(self) -> StringArgument: return copy.deepcopy(self._env_vars) @env_vars.setter - def env_vars(self, value: t.Mapping[str, str]) -> None: + def env_vars(self, value: t.Dict[str, str | None]) -> None: """Set the environment variables. """ self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, scheduler_args) -> BatchArgBuilder: + def _get_arg_builder(self, scheduler_args: StringArgument | None) -> BatchArgBuilder: """ Map the Scheduler to the BatchArgBuilder """ if self._batch_scheduler == SchedulerType.Slurm: @@ -99,9 +105,9 @@ def format_batch_args(self) -> t.List[str]: return self._arg_builder.format_batch_args() def __str__(self) -> str: # pragma: no-cover - string = f"\nScheduler: {self.arg_translator.scheduler_str}" + string = f"\nScheduler: {self.scheduler}" if self.scheduler_args: - string += f"\nScheduler Arguments:\n{fmt_dict(self.scheduler_args)}" + string += f"\nScheduler Arguments:\n{fmt_dict(self.scheduler_args._scheduler_args)}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" return string \ No newline at end of file diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index 8b61d3e7df..22c90fff12 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -25,11 +25,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import typing as t +from smartsim.log import get_logger IntegerArgument = t.Dict[str, t.Optional[int]] StringArgument = t.Dict[str, t.Optional[str]] -def set_check_input(key: str, value: str | None, logger): +logger = get_logger(__name__) + +def set_check_input(key: str, value: str | None) -> None: # TODO check this if not (isinstance(key, str) or isinstance(value, (str, None))): raise TypeError("Argument name should be of type str") diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index b52e4c3ad7..081e2cd662 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -49,15 +49,15 @@ class LaunchSettings(BaseSettings): def __init__( self, launcher: t.Union[LauncherType, str], - launch_args: t.Optional[StringArgument] = None, - env_vars: t.Optional[StringArgument] = None, + launch_args: StringArgument | None = None, + env_vars: StringArgument | None = None, ) -> None: try: self._launcher = LauncherType(launcher) except ValueError: raise ValueError(f"Invalid launcher type: {launcher}") self._arg_builder = self._get_arg_builder(launch_args) - self.env_vars = env_vars or {} + self.env_vars = copy.deepcopy(env_vars) if env_vars else {} @property def launcher(self) -> str: @@ -81,18 +81,18 @@ def launch_args(self, args: t.Mapping[str, str]) -> None: self.launch_args.set(k, v) @property - def env_vars(self) -> StringArgument: + def env_vars(self) -> t.Mapping[str, str | None]: """Return an immutable list of attached environment variables. """ return copy.deepcopy(self._env_vars) @env_vars.setter - def env_vars(self, value: t.Mapping[str, str]) -> None: + def env_vars(self, value: t.Dict[str, str]) -> None: """Set the environment variables. """ self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, launch_args) -> LaunchArgBuilder: + def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder: """ Map the Launcher to the LaunchArgBuilder """ if self._launcher == LauncherType.Slurm: @@ -116,7 +116,7 @@ def _get_arg_builder(self, launch_args) -> LaunchArgBuilder: else: raise ValueError(f"Invalid launcher type: {self._launcher}") - def update_env(self, env_vars: StringArgument) -> None: + def update_env(self, env_vars: t.Dict[str, str | None]) -> None: """Update the job environment variables To fully inherit the current user environment, add the @@ -140,7 +140,7 @@ def format_env_vars(self) -> t.Union[t.List[str],None]: """Build bash compatible environment variable string for Slurm :returns: the formatted string of environment variables """ - return self._arg_builder.format_env_vars(self.env_vars) + return self._arg_builder.format_env_vars(self._env_vars) def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: """Build environment variable string for Slurm @@ -149,7 +149,7 @@ def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: for more information on this, see the slurm documentation for srun :returns: the formatted string of environment variables """ - return self._arg_builder.format_comma_sep_env_vars(self.env_vars) + return self._arg_builder.format_comma_sep_env_vars(self._env_vars) def format_launch_args(self) -> t.Union[t.List[str],None]: """Return formatted launch arguments diff --git a/smartsim/settings/translators/batch/lsf.py b/smartsim/settings/translators/batch/lsf.py index a6ed8dc525..7a435e5ae2 100644 --- a/smartsim/settings/translators/batch/lsf.py +++ b/smartsim/settings/translators/batch/lsf.py @@ -29,7 +29,8 @@ import typing as t from ..batchArgBuilder import BatchArgBuilder from ...common import StringArgument -from smartsim.log import get_logger +from smartsim.log import get_logger +from ...batchCommand import SchedulerType logger = get_logger(__name__) @@ -37,10 +38,15 @@ class BsubBatchArgBuilder(BatchArgBuilder): def __init__( self, - scheduler_args: StringArgument, + scheduler_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(scheduler_args) + def scheduler_str(self) -> str: + """ Get the string representation of the scheduler + """ + return SchedulerType.Lsf.value + def set_walltime(self, walltime: str) -> None: """Set the walltime diff --git a/smartsim/settings/translators/batch/pbs.py b/smartsim/settings/translators/batch/pbs.py index c8b9f9289d..770f8b401c 100644 --- a/smartsim/settings/translators/batch/pbs.py +++ b/smartsim/settings/translators/batch/pbs.py @@ -31,6 +31,7 @@ from ..batchArgBuilder import BatchArgBuilder from ....error import SSConfigError from ...common import StringArgument +from ...batchCommand import SchedulerType from smartsim.log import get_logger logger = get_logger(__name__) @@ -38,10 +39,15 @@ class QsubBatchArgBuilder(BatchArgBuilder): def __init__( self, - scheduler_args: StringArgument, + scheduler_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(scheduler_args) + def scheduler_str(self) -> str: + """ Get the string representation of the scheduler + """ + return SchedulerType.Pbs.value + def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job diff --git a/smartsim/settings/translators/batch/slurm.py b/smartsim/settings/translators/batch/slurm.py index c26a7bf9db..bc252e43e3 100644 --- a/smartsim/settings/translators/batch/slurm.py +++ b/smartsim/settings/translators/batch/slurm.py @@ -30,6 +30,7 @@ import typing as t from ..batchArgBuilder import BatchArgBuilder from ...common import StringArgument +from ...batchCommand import SchedulerType from smartsim.log import get_logger logger = get_logger(__name__) @@ -38,10 +39,15 @@ class SlurmBatchArgBuilder(BatchArgBuilder): def __init__( self, - scheduler_args: StringArgument, + scheduler_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(scheduler_args) + def scheduler_str(self) -> str: + """ Get the string representation of the scheduler + """ + return SchedulerType.Slurm.value + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job diff --git a/smartsim/settings/translators/batchArgBuilder.py b/smartsim/settings/translators/batchArgBuilder.py index 723cc5421b..6105a1c73c 100644 --- a/smartsim/settings/translators/batchArgBuilder.py +++ b/smartsim/settings/translators/batchArgBuilder.py @@ -41,9 +41,15 @@ class BatchArgBuilder(ABC): the input parameter to a properly formatted launcher argument. """ - def __init__(self, scheduler_args) -> None: + def __init__(self, scheduler_args: t.Dict[str, str | None] | None) -> None: self._scheduler_args = copy.deepcopy(scheduler_args) or {} + @abstractmethod + def scheduler_str(self) -> str: + """ Get the string representation of the launcher + """ + pass + @abstractmethod def set_account(self, account: str) -> None: """Set the account for this batch job @@ -88,7 +94,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: pass @abstractmethod - def format_batch_args(self, batch_args: t.Dict[str, t.Union[str,int,float,None]]) -> t.List[str]: + def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview :return: batch arguments for Sbatch diff --git a/smartsim/settings/translators/launch/alps.py b/smartsim/settings/translators/launch/alps.py index 68ed18db00..bf8eeea164 100644 --- a/smartsim/settings/translators/launch/alps.py +++ b/smartsim/settings/translators/launch/alps.py @@ -28,6 +28,7 @@ from ..launchArgBuilder import LaunchArgBuilder import typing as t from ...common import set_check_input, StringArgument +from ...launchCommand import LauncherType from smartsim.log import get_logger logger = get_logger(__name__) @@ -36,7 +37,7 @@ class AprunArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) @@ -45,6 +46,11 @@ def _reserved_launch_args(self) -> set[str]: """ return {"wdir"} + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Alps.value + def set_cpus_per_task(self, cpus_per_task: int) -> None: """Set the number of cpus to use per task @@ -154,7 +160,7 @@ def set_verbose_launch(self, verbose: bool) -> None: else: self._launch_args.pop("debug", None) - def set_quiet_launch(self, quiet: bool) -> t.Union[t.Dict[str,None],None]: + def set_quiet_launch(self, quiet: bool) -> None: """Set the job to run in quiet mode This sets ``--quiet`` @@ -199,7 +205,7 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._reserved_launch_args(): logger.warning( ( diff --git a/smartsim/settings/translators/launch/dragon.py b/smartsim/settings/translators/launch/dragon.py index fd354f5b77..24b4fe0fd8 100644 --- a/smartsim/settings/translators/launch/dragon.py +++ b/smartsim/settings/translators/launch/dragon.py @@ -29,7 +29,8 @@ import typing as t from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input -from smartsim.log import get_logger +from smartsim.log import get_logger +from ...launchCommand import LauncherType logger = get_logger(__name__) @@ -37,10 +38,15 @@ class DragonArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Dragon.value + def set_nodes(self, nodes: int) -> None: """Set the number of nodes @@ -58,7 +64,7 @@ def set_tasks_per_node(self, tasks_per_node: int) -> None: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/local.py b/smartsim/settings/translators/launch/local.py index 432a18a6cb..0ec3a7eb58 100644 --- a/smartsim/settings/translators/launch/local.py +++ b/smartsim/settings/translators/launch/local.py @@ -29,7 +29,8 @@ import typing as t from ..launchArgBuilder import LaunchArgBuilder from smartsim.log import get_logger -from ...common import StringArgument, set_check_input +from ...common import StringArgument, set_check_input +from ...launchCommand import LauncherType logger = get_logger(__name__) @@ -37,10 +38,15 @@ class LocalArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Local.value + def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: """Build environment variable string @@ -68,7 +74,7 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/lsf.py b/smartsim/settings/translators/launch/lsf.py index 3fa27044bf..51753a74dd 100644 --- a/smartsim/settings/translators/launch/lsf.py +++ b/smartsim/settings/translators/launch/lsf.py @@ -29,7 +29,8 @@ import typing as t from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input -from smartsim.log import get_logger +from smartsim.log import get_logger +from ...launchCommand import LauncherType logger = get_logger(__name__) @@ -37,10 +38,15 @@ class JsrunArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Lsf.value + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ @@ -102,7 +108,7 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._reserved_launch_args(): logger.warning( ( diff --git a/smartsim/settings/translators/launch/mpi.py b/smartsim/settings/translators/launch/mpi.py index 05aec2506b..f83e9f4d51 100644 --- a/smartsim/settings/translators/launch/mpi.py +++ b/smartsim/settings/translators/launch/mpi.py @@ -28,7 +28,7 @@ import typing as t from ..launchArgBuilder import LaunchArgBuilder -from ...common import StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType from smartsim.log import get_logger @@ -38,7 +38,7 @@ class _BaseMPIArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) @@ -204,7 +204,7 @@ def format_launch_args(self) -> t.List[str]: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._reserved_launch_args(): logger.warning( ( @@ -221,22 +221,37 @@ class MpiArgBuilder(_BaseMPIArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Mpirun.value + class MpiexecArgBuilder(_BaseMPIArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Mpiexec.value + class OrteArgBuilder(_BaseMPIArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: - super().__init__(launch_args) \ No newline at end of file + super().__init__(launch_args) + + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Orterun.value \ No newline at end of file diff --git a/smartsim/settings/translators/launch/pals.py b/smartsim/settings/translators/launch/pals.py index e23629e936..8c46594dd6 100644 --- a/smartsim/settings/translators/launch/pals.py +++ b/smartsim/settings/translators/launch/pals.py @@ -29,7 +29,8 @@ import typing as t from ..launchArgBuilder import LaunchArgBuilder from ...common import StringArgument, set_check_input -from smartsim.log import get_logger +from smartsim.log import get_logger +from ...launchCommand import LauncherType logger = get_logger(__name__) @@ -37,10 +38,15 @@ class PalsMpiexecArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Pals.value + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ @@ -136,7 +142,7 @@ def format_launch_args(self) -> t.List[str]: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._reserved_launch_args(): logger.warning( ( diff --git a/smartsim/settings/translators/launch/slurm.py b/smartsim/settings/translators/launch/slurm.py index 314fdea20d..59c2c7de3b 100644 --- a/smartsim/settings/translators/launch/slurm.py +++ b/smartsim/settings/translators/launch/slurm.py @@ -31,7 +31,8 @@ import os from ..launchArgBuilder import LaunchArgBuilder from ...common import IntegerArgument, StringArgument, set_check_input -from smartsim.log import get_logger +from smartsim.log import get_logger +from ...launchCommand import LauncherType logger = get_logger(__name__) @@ -39,10 +40,15 @@ class SlurmArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: StringArgument, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + return LauncherType.Slurm.value + def _reserved_launch_args(self) -> set[str]: """ Return reserved launch arguments. """ @@ -300,7 +306,7 @@ def _check_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> None: def set(self, key: str, value: str | None) -> None: """ Set the launch arguments """ - set_check_input(key,value,logger) + set_check_input(key,value) if key in self._reserved_launch_args(): logger.warning( ( diff --git a/smartsim/settings/translators/launchArgBuilder.py b/smartsim/settings/translators/launchArgBuilder.py index 5b5ddc9941..eb85347485 100644 --- a/smartsim/settings/translators/launchArgBuilder.py +++ b/smartsim/settings/translators/launchArgBuilder.py @@ -27,6 +27,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +import typing as t import copy from smartsim.log import get_logger @@ -39,11 +40,43 @@ class LaunchArgBuilder(ABC): responsibility of child classes for each launcher to translate the input parameter to a properly formatted launcher argument. """ - def __init__(self, launch_args) -> None: + def __init__(self, launch_args: t.Dict[str, str | None] | None) -> None: self._launch_args = copy.deepcopy(launch_args) or {} + @abstractmethod + def launcher_str(self) -> str: + """ Get the string representation of the launcher + """ + pass + @abstractmethod def set(self, arg: str, val: str | None) -> None: """ Set the launch arguments """ - pass \ No newline at end of file + pass + + def format_launch_args(self) -> t.Union[t.List[str],None]: + """ Build formatted launch arguments + """ + logger.warning(f"format_launcher_args() not supported for {self.launcher_str()}.") + return None + + def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.Tuple[str, t.List[str]],None]: + """Build environment variable string for Slurm + + Slurm takes exports in comma separated lists + the list starts with all as to not disturb the rest of the environment + for more information on this, see the slurm documentation for srun + + :returns: the formatted string of environment variables + """ + logger.warning(f"format_comma_sep_env_vars() not supported for {self.launcher_str()}.") + return None + + def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + """Build bash compatible environment variable string for Slurm + + :returns: the formatted string of environment variables + """ + logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") + return None \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 6fbadbf85e..5c7e9e730e 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -2,7 +2,6 @@ from smartsim.settings.translators.launch.dragon import DragonArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest -import logging @pytest.mark.parametrize( "function,value,result,flag", diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index badb2384aa..890e54430c 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -2,7 +2,6 @@ from smartsim.settings.translators.launch.local import LocalArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest -import logging # TODO complete after launch args retrieval def test_launch_args_input_mutation(): diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 5c6ea12ddb..a566ef9776 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -1,7 +1,6 @@ from smartsim.settings import LaunchSettings from smartsim.settings.translators.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder import pytest -import logging import itertools from smartsim.settings.launchCommand import LauncherType diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index b64fdbe54b..be4fa75a24 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -16,6 +16,7 @@ ) def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.Pals) + assert palsLauncher.launch_args == PalsMpiexecArgBuilder getattr(palsLauncher.launch_args, function)(*value) assert palsLauncher.launch_args._launch_args[flag] == result assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index 34bcb38ec8..5a6bfa7f11 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -17,6 +17,7 @@ ) def test_create_pbs_batch(function, value, flag, result): pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.Pbs) + assert pbsScheduler.scheduler_args == QsubBatchArgBuilder getattr(pbsScheduler.scheduler_args, function)(*value) assert pbsScheduler.scheduler_args._scheduler_args[flag] == result diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 3ccf72388e..b69acde30f 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -2,7 +2,6 @@ from smartsim.settings.translators.launch.slurm import SlurmArgBuilder from smartsim.settings.launchCommand import LauncherType import pytest -import logging @pytest.mark.parametrize( "function,value,result,flag", From 9d6d06d14f27fd8a9d39196b18045e6539b94bf7 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 6 Jun 2024 15:34:25 -0500 Subject: [PATCH 34/43] str tests --- smartsim/_core/utils/helpers.py | 2 +- tests/temp_tests/test_settings/test_alpsLauncher.py | 5 +++++ .../temp_tests/test_settings/test_dragonLauncher.py | 5 +++++ .../temp_tests/test_settings/test_localLauncher.py | 5 +++++ tests/temp_tests/test_settings/test_lsfLauncher.py | 5 +++++ tests/temp_tests/test_settings/test_lsfScheduler.py | 5 +++++ tests/temp_tests/test_settings/test_mpiLauncher.py | 13 +++++++++++++ tests/temp_tests/test_settings/test_palsLauncher.py | 7 ++++++- tests/temp_tests/test_settings/test_pbsScheduler.py | 7 ++++++- .../temp_tests/test_settings/test_slurmLauncher.py | 5 +++++ .../temp_tests/test_settings/test_slurmScheduler.py | 5 +++++ 11 files changed, 61 insertions(+), 3 deletions(-) diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index def6220a82..9418981ceb 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -87,7 +87,7 @@ def check_dev_log_level() -> bool: return lvl == "developer" -def fmt_dict(value: t.Dict[str, t.Any]) -> str: +def fmt_dict(value: t.Dict[str, t.Any] | t.Mapping[str, t.Any]) -> str: fmt_str = "" for k, v in value.items(): fmt_str += "\t" + str(k) + " = " + str(v) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index fd4f05b30f..26937aff4f 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -3,6 +3,11 @@ from smartsim.settings.translators.launch.alps import AprunArgBuilder from smartsim.settings.launchCommand import LauncherType +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) + assert alpsLauncher.launch_args.launcher_str() == LauncherType.Alps.value + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 5c7e9e730e..f07c1ce5e1 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -3,6 +3,11 @@ from smartsim.settings.launchCommand import LauncherType import pytest +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + ls = LaunchSettings(launcher=LauncherType.Dragon) + assert ls.launch_args.launcher_str() == LauncherType.Dragon.value + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 890e54430c..477815d1f2 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -3,6 +3,11 @@ from smartsim.settings.launchCommand import LauncherType import pytest +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + ls = LaunchSettings(launcher=LauncherType.Local) + assert ls.launch_args.launcher_str() == LauncherType.Local.value + # TODO complete after launch args retrieval def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index b679c9845f..c9260ec58a 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -3,6 +3,11 @@ from smartsim.settings.launchCommand import LauncherType import pytest +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + ls = LaunchSettings(launcher=LauncherType.Lsf) + assert ls.launch_args.launcher_str() == LauncherType.Lsf.value + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/test_settings/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py index baf3b2e74d..4cc21df1a6 100644 --- a/tests/temp_tests/test_settings/test_lsfScheduler.py +++ b/tests/temp_tests/test_settings/test_lsfScheduler.py @@ -2,6 +2,11 @@ import pytest from smartsim.settings.batchCommand import SchedulerType +def test_scheduler_str(): + """Ensure scheduler_str returns appropriate value""" + bs = BatchSettings(batch_scheduler=SchedulerType.Lsf) + assert bs.scheduler_args.scheduler_str() == SchedulerType.Lsf.value + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index a566ef9776..96e97aadcc 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -4,6 +4,19 @@ import itertools from smartsim.settings.launchCommand import LauncherType +@pytest.mark.parametrize( + "launcher", + [ + pytest.param(LauncherType.Mpirun, id="launcher_str_mpirun"), + pytest.param(LauncherType.Mpiexec, id="launcher_str_mpiexec"), + pytest.param(LauncherType.Orterun, id="launcher_str_orterun"), + ], +) +def test_launcher_str(launcher): + """Ensure launcher_str returns appropriate value""" + ls = LaunchSettings(launcher=launcher) + assert ls.launch_args.launcher_str() == launcher.value + @pytest.mark.parametrize( "l,function,value,result,flag", [ diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index be4fa75a24..15e28a4780 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -3,6 +3,11 @@ from smartsim.settings.launchCommand import LauncherType import pytest +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + ls = LaunchSettings(launcher=LauncherType.Pals) + assert ls.launch_args.launcher_str() == LauncherType.Pals.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -16,7 +21,7 @@ ) def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.Pals) - assert palsLauncher.launch_args == PalsMpiexecArgBuilder + assert isinstance(palsLauncher.launch_args,PalsMpiexecArgBuilder) getattr(palsLauncher.launch_args, function)(*value) assert palsLauncher.launch_args._launch_args[flag] == result assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index 5a6bfa7f11..758c16848c 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -3,6 +3,11 @@ from smartsim.settings.batchCommand import SchedulerType import pytest +def test_scheduler_str(): + """Ensure scheduler_str returns appropriate value""" + bs = BatchSettings(batch_scheduler=SchedulerType.Pbs) + assert bs.scheduler_args.scheduler_str() == SchedulerType.Pbs.value + @pytest.mark.parametrize( "function,value,result,flag", [ @@ -17,7 +22,7 @@ ) def test_create_pbs_batch(function, value, flag, result): pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.Pbs) - assert pbsScheduler.scheduler_args == QsubBatchArgBuilder + assert isinstance(pbsScheduler.scheduler_args, QsubBatchArgBuilder) getattr(pbsScheduler.scheduler_args, function)(*value) assert pbsScheduler.scheduler_args._scheduler_args[flag] == result diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index b69acde30f..1fa18c00e4 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -3,6 +3,11 @@ from smartsim.settings.launchCommand import LauncherType import pytest +def test_launcher_str(): + """Ensure launcher_str returns appropriate value""" + ls = LaunchSettings(launcher=LauncherType.Slurm) + assert ls.launch_args.launcher_str() == LauncherType.Slurm.value + @pytest.mark.parametrize( "function,value,result,flag", [ diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index 9cd6d4725d..f91c01eee8 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -3,6 +3,11 @@ from smartsim.settings.batchCommand import SchedulerType import pytest +def test_scheduler_str(): + """Ensure scheduler_str returns appropriate value""" + bs = BatchSettings(batch_scheduler=SchedulerType.Slurm) + assert bs.scheduler_args.scheduler_str() == SchedulerType.Slurm.value + @pytest.mark.parametrize( "function,value,result,flag", [ From 603bd0ab83216d5323a144526461f1a1d91ab284 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 11 Jun 2024 16:59:18 -0500 Subject: [PATCH 35/43] pushing --- .../test_core/test_commands/test_commandList.py | 6 +++--- .../test_core/test_commands/test_launchCommands.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 945c226cfe..1779191de7 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -28,9 +28,9 @@ from smartsim._core.commands.command import Command from smartsim.settings.launchCommand import LauncherType -salloc_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) -srun_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["srun", "-n", "1"]) -sacct_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["sacct", "--user"]) +salloc_cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) +srun_cmd = Command(launcher=LauncherType.Slurm, command=["srun", "-n", "1"]) +sacct_cmd = Command(launcher=LauncherType.Slurm, command=["sacct", "--user"]) def test_command_init(): cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) diff --git a/tests/temp_tests/test_core/test_commands/test_launchCommands.py b/tests/temp_tests/test_core/test_commands/test_launchCommands.py index 26cfbca1f9..d76887e46a 100644 --- a/tests/temp_tests/test_core/test_commands/test_launchCommands.py +++ b/tests/temp_tests/test_core/test_commands/test_launchCommands.py @@ -3,15 +3,15 @@ from smartsim._core.commands.launchCommands import LaunchCommands from smartsim.settings.launchCommand import LauncherType -pre_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["pre", "cmd"]) -launch_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["launch", "cmd"]) -post_cmd = Command(launcher=LauncherType.SlurmLauncher, command=["post", "cmd"]) +pre_cmd = Command(launcher=LauncherType.Slurm, command=["pre", "cmd"]) +launch_cmd = Command(launcher=LauncherType.Slurm, command=["launch", "cmd"]) +post_cmd = Command(launcher=LauncherType.Slurm, command=["post", "cmd"]) pre_commands_list = CommandList(commands=[pre_cmd]) launch_command_list = CommandList(commands=[launch_cmd]) post_command_list = CommandList(commands=[post_cmd]) def test_launchCommand_init(): launch_cmd = LaunchCommands(prelaunch_commands=pre_commands_list,launch_commands=launch_command_list,postlaunch_commands=post_command_list) - assert launch_cmd.prelaunch_command_maps == pre_commands_list - assert launch_cmd.launch_command_maps == launch_command_list - assert launch_cmd.postlaunch_command_maps == post_command_list + assert launch_cmd.prelaunch_command == pre_commands_list + assert launch_cmd.launch_command == launch_command_list + assert launch_cmd.postlaunch_command == post_command_list From bbc13321f5bfdd136080704ea8e119e888440f97 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 11 Jun 2024 17:16:21 -0500 Subject: [PATCH 36/43] make style chnages --- smartsim/_core/commands/command.py | 40 +++--- smartsim/_core/commands/commandList.py | 34 +++-- smartsim/_core/commands/launchCommands.py | 19 +-- smartsim/_core/launcher/step/localStep.py | 3 +- smartsim/_core/launcher/step/lsfStep.py | 4 +- smartsim/_core/launcher/step/mpiStep.py | 4 +- smartsim/_core/launcher/step/step.py | 1 - smartsim/database/orchestrator.py | 7 +- smartsim/experiment.py | 2 +- smartsim/settings/__init__.py | 85 ++++++++---- smartsim/settings/batchCommand.py | 6 +- smartsim/settings/batchSettings.py | 39 +++--- smartsim/settings/common.py | 5 +- smartsim/settings/launchCommand.py | 6 +- smartsim/settings/launchSettings.py | 56 ++++---- smartsim/settings/translators/__init__.py | 7 +- smartsim/settings/translators/batch/lsf.py | 21 +-- smartsim/settings/translators/batch/pbs.py | 36 ++--- smartsim/settings/translators/batch/slurm.py | 27 ++-- .../settings/translators/batchArgBuilder.py | 14 +- smartsim/settings/translators/launch/alps.py | 50 +++---- .../settings/translators/launch/dragon.py | 26 ++-- smartsim/settings/translators/launch/local.py | 26 ++-- smartsim/settings/translators/launch/lsf.py | 33 ++--- smartsim/settings/translators/launch/mpi.py | 83 ++++++------ smartsim/settings/translators/launch/pals.py | 51 +++---- smartsim/settings/translators/launch/slurm.py | 103 ++++++++------- .../settings/translators/launchArgBuilder.py | 43 +++--- smartsim/wlm/slurm.py | 3 + tests/temp_tests/steps_tests.py | 71 +++++++--- .../test_core/test_commands/test_command.py | 29 ++-- .../test_commands/test_commandList.py | 28 ++-- .../test_commands/test_launchCommands.py | 9 +- .../test_settings/test_alpsLauncher.py | 111 ++++++++++++---- .../test_settings/test_batchSettings.py | 35 +++-- .../test_settings/test_dragonLauncher.py | 17 ++- .../test_settings/test_launchSettings.py | 76 ++++++----- .../test_settings/test_localLauncher.py | 30 +++-- .../test_settings/test_lsfLauncher.py | 23 ++-- .../test_settings/test_lsfScheduler.py | 38 ++++-- .../test_settings/test_mpiLauncher.py | 124 ++++++++++++++---- .../test_settings/test_palsLauncher.py | 47 +++++-- .../test_settings/test_pbsScheduler.py | 50 ++++--- .../test_settings/test_slurmLauncher.py | 121 +++++++++++++---- .../test_settings/test_slurmScheduler.py | 67 +++++++--- 45 files changed, 1072 insertions(+), 638 deletions(-) diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 27ced30663..d89aa41ad9 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -24,16 +24,17 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import typing as t from collections.abc import MutableSequence + from ...settings.launchCommand import LauncherType -import typing as t + class Command(MutableSequence[str]): - """Basic container for command information - """ - def __init__(self, launcher: LauncherType, command:t.List[str]) -> None: - """Command constructor - """ + """Basic container for command information""" + + def __init__(self, launcher: LauncherType, command: t.List[str]) -> None: + """Command constructor""" self._launcher = launcher self._command = command @@ -43,40 +44,35 @@ def launcher(self) -> LauncherType: Return a reference to the LauncherType. """ return self._launcher - + @property def command(self) -> t.List[str]: """Get the command list. Return a reference to the command list. """ return self._command - + def __getitem__(self, idx: int) -> str: - """Get the command at the specified index. - """ + """Get the command at the specified index.""" return self._command[idx] - + def __setitem__(self, idx: int, value: str) -> None: - """Set the command at the specified index. - """ + """Set the command at the specified index.""" self._command[idx] = value def __delitem__(self, idx: int) -> None: - """Delete the command at the specified index. - """ + """Delete the command at the specified index.""" del self._command[idx] def __len__(self) -> int: - """Get the length of the command list. - """ + """Get the length of the command list.""" return len(self._command) def insert(self, idx: int, value: str) -> None: - """Insert a command at the specified index. - """ + """Insert a command at the specified index.""" self._command.insert(idx, value) - - def __str__(self) -> str: # pragma: no cover + + def __str__(self) -> str: # pragma: no cover string = f"\nLauncher: {self.launcher.value}\n" string += f"Command: {' '.join(str(cmd) for cmd in self.command)}" - return string \ No newline at end of file + return string diff --git a/smartsim/_core/commands/commandList.py b/smartsim/_core/commands/commandList.py index 0fb0d75764..08b95bbfd1 100644 --- a/smartsim/_core/commands/commandList.py +++ b/smartsim/_core/commands/commandList.py @@ -24,16 +24,17 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import typing as t from collections.abc import MutableSequence + from .command import Command -import typing as t + class CommandList(MutableSequence[Command]): - """Container for a Sequence of Command objects - """ + """Container for a Sequence of Command objects""" + def __init__(self, commands: t.Union[Command, t.List[Command]]): - """CommandList constructor - """ + """CommandList constructor""" if isinstance(commands, Command): commands = [commands] self._commands: t.List[Command] = list(commands) @@ -44,35 +45,30 @@ def commands(self) -> t.List[Command]: Return a reference to the Command list. """ return self._commands - + def __getitem__(self, idx: int) -> Command: - """Get the Command at the specified index. - """ + """Get the Command at the specified index.""" return self._commands[idx] - + def __setitem__(self, idx: int, value: Command) -> None: - """Set the Command at the specified index. - """ + """Set the Command at the specified index.""" self._commands[idx] = value def __delitem__(self, idx: int) -> None: - """Delete the Command at the specified index. - """ + """Delete the Command at the specified index.""" del self._commands[idx] def __len__(self) -> int: - """Get the length of the Command list. - """ + """Get the length of the Command list.""" return len(self._commands) def insert(self, idx: int, value: Command) -> None: - """Insert a Command at the specified index. - """ + """Insert a Command at the specified index.""" self._commands.insert(idx, value) - def __str__(self) -> str: # pragma: no cover + def __str__(self) -> str: # pragma: no cover string = "\n\nCommand List:\n\n" for counter, cmd in enumerate(self.commands): string += f"CommandList index {counter} value:" string += f"{cmd}\n\n" - return string \ No newline at end of file + return string diff --git a/smartsim/_core/commands/launchCommands.py b/smartsim/_core/commands/launchCommands.py index 96d1203e1b..53d3da6f84 100644 --- a/smartsim/_core/commands/launchCommands.py +++ b/smartsim/_core/commands/launchCommands.py @@ -1,17 +1,18 @@ from .commandList import CommandList + class LaunchCommands: """Container for aggregating prelaunch commands (e.g. file system operations), launch commands, and postlaunch commands """ + def __init__( - self, - prelaunch_commands: CommandList, - launch_commands: CommandList, - postlaunch_commands: CommandList, - ) -> None: - """LaunchCommand constructor - """ + self, + prelaunch_commands: CommandList, + launch_commands: CommandList, + postlaunch_commands: CommandList, + ) -> None: + """LaunchCommand constructor""" self._prelaunch_commands = prelaunch_commands self._launch_commands = launch_commands self._postlaunch_commands = postlaunch_commands @@ -37,7 +38,7 @@ def postlaunch_command(self) -> CommandList: """ return self._postlaunch_commands - def __str__(self) -> str: # pragma: no cover + def __str__(self) -> str: # pragma: no cover string = "\n\nPrelaunch Command List:\n" for _, pre_cmd in enumerate(self.prelaunch_command): string += f"{pre_cmd}\n" @@ -47,4 +48,4 @@ def __str__(self) -> str: # pragma: no cover string += "\n\nPostlaunch Command List:\n" for _, post_cmd in enumerate(self.postlaunch_command): string += f"{post_cmd}\n" - return string \ No newline at end of file + return string diff --git a/smartsim/_core/launcher/step/localStep.py b/smartsim/_core/launcher/step/localStep.py index 14ebba2aa3..49666a2059 100644 --- a/smartsim/_core/launcher/step/localStep.py +++ b/smartsim/_core/launcher/step/localStep.py @@ -29,8 +29,7 @@ import typing as t from ....entity import Application, FSNode -from ....settings import Singularity -from ....settings import RunSettings +from ....settings import RunSettings, Singularity from .step import Step, proxyable_launch_cmd diff --git a/smartsim/_core/launcher/step/lsfStep.py b/smartsim/_core/launcher/step/lsfStep.py index f8674c8b04..372e21c81b 100644 --- a/smartsim/_core/launcher/step/lsfStep.py +++ b/smartsim/_core/launcher/step/lsfStep.py @@ -31,9 +31,7 @@ from ....entity import Application, FSNode from ....error import AllocationError from ....log import get_logger -from ....settings import BsubBatchSettings, JsrunSettings -from ....settings import RunSettings - +from ....settings import BsubBatchSettings, JsrunSettings, RunSettings from .step import Step logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/mpiStep.py b/smartsim/_core/launcher/step/mpiStep.py index ac1b113944..8806649c84 100644 --- a/smartsim/_core/launcher/step/mpiStep.py +++ b/smartsim/_core/launcher/step/mpiStep.py @@ -32,9 +32,7 @@ from ....entity import Application, FSNode from ....error import AllocationError, SmartSimError from ....log import get_logger -from ....settings import MpiexecSettings, MpirunSettings, OrterunSettings -from ....settings import RunSettings - +from ....settings import MpiexecSettings, MpirunSettings, OrterunSettings, RunSettings from .step import Step, proxyable_launch_cmd logger = get_logger(__name__) diff --git a/smartsim/_core/launcher/step/step.py b/smartsim/_core/launcher/step/step.py index eb15dc9776..dafe94336d 100644 --- a/smartsim/_core/launcher/step/step.py +++ b/smartsim/_core/launcher/step/step.py @@ -40,7 +40,6 @@ from ....entity import Application, Ensemble, FSNode from ....log import get_logger from ....settings import RunSettings, SettingsBase - from ...utils.helpers import encode_cmd, get_base_36_repr from ..colocated import write_colocated_launch_script diff --git a/smartsim/database/orchestrator.py b/smartsim/database/orchestrator.py index dd9014832d..446adfab08 100644 --- a/smartsim/database/orchestrator.py +++ b/smartsim/database/orchestrator.py @@ -52,6 +52,7 @@ from ..servertype import CLUSTERED, STANDALONE from ..settings import ( AprunSettings, + BatchSettings, BsubBatchSettings, JsrunSettings, MpiexecSettings, @@ -59,12 +60,12 @@ OrterunSettings, PalsMpiexecSettings, QsubBatchSettings, + RunSettings, SbatchSettings, SrunSettings, + create_batch_settings, + create_run_settings, ) - -from ..settings import BatchSettings, RunSettings -from ..settings import create_batch_settings, create_run_settings from ..wlm import detect_launcher logger = get_logger(__name__) diff --git a/smartsim/experiment.py b/smartsim/experiment.py index d6d3f57ec3..087f6664d6 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -48,7 +48,7 @@ ) from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer -from .settings import Container, RunSettings, BatchSettings +from .settings import BatchSettings, Container, RunSettings from .wlm import detect_launcher logger = get_logger(__name__) diff --git a/smartsim/settings/__init__.py b/smartsim/settings/__init__.py index b8f40b03c3..b426a266e2 100644 --- a/smartsim/settings/__init__.py +++ b/smartsim/settings/__init__.py @@ -24,32 +24,73 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .launchSettings import LaunchSettings from .baseSettings import BaseSettings from .batchSettings import BatchSettings +from .launchSettings import LaunchSettings + +__all__ = ["LaunchSettings", "BaseSettings", "BatchSettings"] -__all__ = [ - "LaunchSettings", - "BaseSettings", - "BatchSettings" -] # TODO Mock imports for compiling tests -class DragonRunSettings: pass -class QsubBatchSettings: pass -class SbatchSettings: pass -class Singularity: pass -class SettingsBase: pass -class AprunSettings: pass -class RunSettings: pass -class OrterunSettings: pass -class MpirunSettings: pass -class MpiexecSettings: pass -class JsrunSettings: pass -class BsubBatchSettings: pass -class PalsMpiexecSettings: pass -class SrunSettings: pass -class Container: pass +class DragonRunSettings: + pass + + +class QsubBatchSettings: + pass + + +class SbatchSettings: + pass + + +class Singularity: + pass + + +class SettingsBase: + pass + + +class AprunSettings: + pass + + +class RunSettings: + pass + + +class OrterunSettings: + pass + + +class MpirunSettings: + pass + + +class MpiexecSettings: + pass + + +class JsrunSettings: + pass + + +class BsubBatchSettings: + pass + + +class PalsMpiexecSettings: + pass + + +class SrunSettings: + pass + + +class Container: + pass + def create_batch_settings() -> None: ... -def create_run_settings() -> None: ... \ No newline at end of file +def create_run_settings() -> None: ... diff --git a/smartsim/settings/batchCommand.py b/smartsim/settings/batchCommand.py index 18aef8642f..8f3b0c89d5 100644 --- a/smartsim/settings/batchCommand.py +++ b/smartsim/settings/batchCommand.py @@ -26,10 +26,12 @@ from enum import Enum + class SchedulerType(Enum): - """ Schedulers that are supported by + """Schedulers that are supported by SmartSim. """ + Slurm = "slurm" Pbs = "pbs" - Lsf = "lsf" \ No newline at end of file + Lsf = "lsf" diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 6a2bdd8fe2..5478069120 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -25,21 +25,24 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations -import typing as t + import copy +import typing as t from smartsim.log import get_logger + from .._core.utils.helpers import fmt_dict -from .common import StringArgument +from .baseSettings import BaseSettings from .batchCommand import SchedulerType +from .common import StringArgument +from .translators import BatchArgBuilder +from .translators.batch.lsf import BsubBatchArgBuilder from .translators.batch.pbs import QsubBatchArgBuilder from .translators.batch.slurm import SlurmBatchArgBuilder -from .translators.batch.lsf import BsubBatchArgBuilder -from .translators import BatchArgBuilder -from .baseSettings import BaseSettings logger = get_logger(__name__) + class BatchSettings(BaseSettings): def __init__( self, @@ -56,38 +59,34 @@ def __init__( @property def scheduler(self) -> str: - """Return the launcher name. - """ + """Return the launcher name.""" return self._batch_scheduler.value @property def batch_scheduler(self) -> str: - """Return the scheduler name. - """ + """Return the scheduler name.""" return self._batch_scheduler.value @property def scheduler_args(self) -> BatchArgBuilder: - """Return the batch argument translator. - """ + """Return the batch argument translator.""" # Is a deep copy needed here? return self._arg_builder @property def env_vars(self) -> StringArgument: - """Return an immutable list of attached environment variables. - """ + """Return an immutable list of attached environment variables.""" return copy.deepcopy(self._env_vars) @env_vars.setter def env_vars(self, value: t.Dict[str, str | None]) -> None: - """Set the environment variables. - """ + """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - def _get_arg_builder(self, scheduler_args: StringArgument | None) -> BatchArgBuilder: - """ Map the Scheduler to the BatchArgBuilder - """ + def _get_arg_builder( + self, scheduler_args: StringArgument | None + ) -> BatchArgBuilder: + """Map the Scheduler to the BatchArgBuilder""" if self._batch_scheduler == SchedulerType.Slurm: return SlurmBatchArgBuilder(scheduler_args) elif self._batch_scheduler == SchedulerType.Lsf: @@ -96,7 +95,7 @@ def _get_arg_builder(self, scheduler_args: StringArgument | None) -> BatchArgBui return QsubBatchArgBuilder(scheduler_args) else: raise ValueError(f"Invalid scheduler type: {self._batch_scheduler}") - + def format_batch_args(self) -> t.List[str]: """Get the formatted batch arguments for a preview @@ -110,4 +109,4 @@ def __str__(self) -> str: # pragma: no-cover string += f"\nScheduler Arguments:\n{fmt_dict(self.scheduler_args._scheduler_args)}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" - return string \ No newline at end of file + return string diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index 22c90fff12..66cf4f7386 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import typing as t + from smartsim.log import get_logger IntegerArgument = t.Dict[str, t.Optional[int]] @@ -32,14 +33,14 @@ logger = get_logger(__name__) + def set_check_input(key: str, value: str | None) -> None: # TODO check this if not (isinstance(key, str) or isinstance(value, (str, None))): - raise TypeError("Argument name should be of type str") + raise TypeError("Argument name should be of type str") if key.startswith("-"): key = key.lstrip("-") logger.warning( "One or more leading `-` characters were provided to the run argument. \ Leading dashes were stripped and the arguments were passed to the run_command." ) - \ No newline at end of file diff --git a/smartsim/settings/launchCommand.py b/smartsim/settings/launchCommand.py index 377bfec972..491f01d867 100644 --- a/smartsim/settings/launchCommand.py +++ b/smartsim/settings/launchCommand.py @@ -26,10 +26,12 @@ from enum import Enum + class LauncherType(Enum): - """ Launchers that are supported by + """Launchers that are supported by SmartSim. """ + Dragon = "dragon" Slurm = "slurm" Pals = "pals" @@ -38,4 +40,4 @@ class LauncherType(Enum): Mpiexec = "mpiexec" Mpirun = "mpirun" Orterun = "orterun" - Lsf = "lsf" \ No newline at end of file + Lsf = "lsf" diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 081e2cd662..9cc133766c 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -25,26 +25,28 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations -import typing as t -import copy +import copy +import typing as t from smartsim.log import get_logger + from .._core.utils.helpers import fmt_dict +from .baseSettings import BaseSettings from .common import StringArgument from .launchCommand import LauncherType +from .translators import LaunchArgBuilder from .translators.launch.alps import AprunArgBuilder +from .translators.launch.dragon import DragonArgBuilder +from .translators.launch.local import LocalArgBuilder from .translators.launch.lsf import JsrunArgBuilder from .translators.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder from .translators.launch.pals import PalsMpiexecArgBuilder -from .translators.launch.slurm import SlurmArgBuilder -from .translators.launch.dragon import DragonArgBuilder -from .translators.launch.local import LocalArgBuilder -from .translators import LaunchArgBuilder -from .baseSettings import BaseSettings +from .translators.launch.slurm import SlurmArgBuilder logger = get_logger(__name__) + class LaunchSettings(BaseSettings): def __init__( self, @@ -58,43 +60,37 @@ def __init__( raise ValueError(f"Invalid launcher type: {launcher}") self._arg_builder = self._get_arg_builder(launch_args) self.env_vars = copy.deepcopy(env_vars) if env_vars else {} - + @property def launcher(self) -> str: - """Return the launcher name. - """ + """Return the launcher name.""" return self._launcher.value - + @property def launch_args(self) -> LaunchArgBuilder: - """Return the launch argument translator. - """ + """Return the launch argument translator.""" # Is a deep copy needed here? return self._arg_builder @launch_args.setter def launch_args(self, args: t.Mapping[str, str]) -> None: - """Update the launch arguments. - """ + """Update the launch arguments.""" self.launch_args._launch_args.clear() for k, v in args.items(): self.launch_args.set(k, v) @property def env_vars(self) -> t.Mapping[str, str | None]: - """Return an immutable list of attached environment variables. - """ + """Return an immutable list of attached environment variables.""" return copy.deepcopy(self._env_vars) @env_vars.setter def env_vars(self, value: t.Dict[str, str]) -> None: - """Set the environment variables. - """ + """Set the environment variables.""" self._env_vars = copy.deepcopy(value) - + def _get_arg_builder(self, launch_args: StringArgument | None) -> LaunchArgBuilder: - """ Map the Launcher to the LaunchArgBuilder - """ + """Map the Launcher to the LaunchArgBuilder""" if self._launcher == LauncherType.Slurm: return SlurmArgBuilder(launch_args) elif self._launcher == LauncherType.Mpiexec: @@ -131,18 +127,16 @@ def update_env(self, env_vars: t.Dict[str, str | None]) -> None: # Coerce env_vars values to str as a convenience to user for env, val in env_vars.items(): if not (isinstance(val, str) and isinstance(env, str)): - raise TypeError( - f"env_vars[{env}] was of type {type(val)}, not str" - ) + raise TypeError(f"env_vars[{env}] was of type {type(val)}, not str") self._env_vars.update(env_vars) - - def format_env_vars(self) -> t.Union[t.List[str],None]: + + def format_env_vars(self) -> t.Union[t.List[str], None]: """Build bash compatible environment variable string for Slurm :returns: the formatted string of environment variables """ return self._arg_builder.format_env_vars(self._env_vars) - def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: + def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]], None]: """Build environment variable string for Slurm Slurm takes exports in comma separated lists the list starts with all as to not disturb the rest of the environment @@ -151,18 +145,18 @@ def format_comma_sep_env_vars(self) -> t.Union[t.Tuple[str, t.List[str]],None]: """ return self._arg_builder.format_comma_sep_env_vars(self._env_vars) - def format_launch_args(self) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str], None]: """Return formatted launch arguments For ``RunSettings``, the run arguments are passed literally with no formatting. :return: list run arguments for these settings """ return self._arg_builder.format_launch_args() - + def __str__(self) -> str: # pragma: no-cover string = f"\nLauncher: {self.launcher}" if self.launch_args._launch_args: string += f"\nLaunch Arguments:\n{fmt_dict(self.launch_args._launch_args)}" if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" - return string \ No newline at end of file + return string diff --git a/smartsim/settings/translators/__init__.py b/smartsim/settings/translators/__init__.py index be4bf2644d..9cfdd5f9c5 100644 --- a/smartsim/settings/translators/__init__.py +++ b/smartsim/settings/translators/__init__.py @@ -24,10 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .launchArgBuilder import LaunchArgBuilder from .batchArgBuilder import BatchArgBuilder +from .launchArgBuilder import LaunchArgBuilder -__all__ = [ - "LaunchArgBuilder", - "BatchArgBuilder" -] \ No newline at end of file +__all__ = ["LaunchArgBuilder", "BatchArgBuilder"] diff --git a/smartsim/settings/translators/batch/lsf.py b/smartsim/settings/translators/batch/lsf.py index 7a435e5ae2..9d44b2c2ad 100644 --- a/smartsim/settings/translators/batch/lsf.py +++ b/smartsim/settings/translators/batch/lsf.py @@ -27,24 +27,25 @@ from __future__ import annotations import typing as t -from ..batchArgBuilder import BatchArgBuilder + +from smartsim.log import get_logger + +from ...batchCommand import SchedulerType from ...common import StringArgument -from smartsim.log import get_logger -from ...batchCommand import SchedulerType +from ..batchArgBuilder import BatchArgBuilder logger = get_logger(__name__) -class BsubBatchArgBuilder(BatchArgBuilder): +class BsubBatchArgBuilder(BatchArgBuilder): def __init__( self, - scheduler_args: t.Dict[str, str | None] | None, + scheduler_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(scheduler_args) def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ + """Get the string representation of the scheduler""" return SchedulerType.Lsf.value def set_walltime(self, walltime: str) -> None: @@ -81,7 +82,7 @@ def set_project(self, project: str) -> None: :param time: project name """ self.set("P", project) - + def set_account(self, account: str) -> None: """Set the project @@ -125,7 +126,7 @@ def set_tasks(self, tasks: int) -> None: def set_queue(self, queue: str) -> None: """Set the queue for this job - + This sets ``-q`` :param queue: The queue to submit the job on @@ -152,4 +153,4 @@ def format_batch_args(self) -> t.List[str]: def set(self, key: str, value: str | None) -> None: # Store custom arguments in the launcher_args - self._scheduler_args[key] = value \ No newline at end of file + self._scheduler_args[key] = value diff --git a/smartsim/settings/translators/batch/pbs.py b/smartsim/settings/translators/batch/pbs.py index 770f8b401c..f04c338106 100644 --- a/smartsim/settings/translators/batch/pbs.py +++ b/smartsim/settings/translators/batch/pbs.py @@ -26,26 +26,28 @@ from __future__ import annotations -from copy import deepcopy import typing as t -from ..batchArgBuilder import BatchArgBuilder +from copy import deepcopy + +from smartsim.log import get_logger + from ....error import SSConfigError -from ...common import StringArgument from ...batchCommand import SchedulerType -from smartsim.log import get_logger +from ...common import StringArgument +from ..batchArgBuilder import BatchArgBuilder + logger = get_logger(__name__) -class QsubBatchArgBuilder(BatchArgBuilder): +class QsubBatchArgBuilder(BatchArgBuilder): def __init__( self, - scheduler_args: t.Dict[str, str | None] | None, + scheduler_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(scheduler_args) def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ + """Get the string representation of the scheduler""" return SchedulerType.Pbs.value def set_nodes(self, num_nodes: int) -> None: @@ -75,7 +77,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be a list of strings") self.set("hostname", ",".join(host_list)) - + def set_walltime(self, walltime: str) -> None: """Set the walltime of the job @@ -88,14 +90,14 @@ def set_walltime(self, walltime: str) -> None: :param walltime: wall time """ self.set("walltime", walltime) - + def set_queue(self, queue: str) -> None: """Set the queue for the batch job :param queue: queue name """ self.set("q", str(queue)) - + def set_ncpus(self, num_cpus: int) -> None: """Set the number of cpus obtained in each node. @@ -106,7 +108,7 @@ def set_ncpus(self, num_cpus: int) -> None: :param num_cpus: number of cpus per node in select """ self.set("ppn", str(num_cpus)) - + def set_account(self, account: str) -> None: """Set the account for this batch job @@ -129,9 +131,7 @@ def format_batch_args(self) -> t.List[str]: return opts @staticmethod - def _sanity_check_resources( - batch_args: t.Dict[str, str | None] - ) -> None: + def _sanity_check_resources(batch_args: t.Dict[str, str | None]) -> None: """Check that only select or nodes was specified in resources Note: For PBS Pro, nodes is equivalent to 'select' and 'place' so @@ -148,7 +148,9 @@ def _sanity_check_resources( "'select' was set using 'set_resource'. Please only specify one." ) - def _create_resource_list(self, batch_args: t.Dict[str, str | None]) -> t.Tuple[t.List[str],t.Dict[str, str | None]]: + def _create_resource_list( + self, batch_args: t.Dict[str, str | None] + ) -> t.Tuple[t.List[str], t.Dict[str, str | None]]: self._sanity_check_resources(batch_args) res = [] @@ -175,4 +177,4 @@ def _create_resource_list(self, batch_args: t.Dict[str, str | None]) -> t.Tuple[ def set(self, key: str, value: str | None) -> None: # Store custom arguments in the launcher_args - self._scheduler_args[key] = value \ No newline at end of file + self._scheduler_args[key] = value diff --git a/smartsim/settings/translators/batch/slurm.py b/smartsim/settings/translators/batch/slurm.py index bc252e43e3..4a24ce6150 100644 --- a/smartsim/settings/translators/batch/slurm.py +++ b/smartsim/settings/translators/batch/slurm.py @@ -28,15 +28,17 @@ import re import typing as t -from ..batchArgBuilder import BatchArgBuilder -from ...common import StringArgument + +from smartsim.log import get_logger + from ...batchCommand import SchedulerType -from smartsim.log import get_logger +from ...common import StringArgument +from ..batchArgBuilder import BatchArgBuilder logger = get_logger(__name__) -class SlurmBatchArgBuilder(BatchArgBuilder): +class SlurmBatchArgBuilder(BatchArgBuilder): def __init__( self, scheduler_args: t.Dict[str, str | None] | None, @@ -44,8 +46,7 @@ def __init__( super().__init__(scheduler_args) def scheduler_str(self) -> str: - """ Get the string representation of the scheduler - """ + """Get the string representation of the scheduler""" return SchedulerType.Slurm.value def set_walltime(self, walltime: str) -> None: @@ -55,7 +56,7 @@ def set_walltime(self, walltime: str) -> None: :param walltime: wall time """ - pattern = r'^\d{2}:\d{2}:\d{2}$' + pattern = r"^\d{2}:\d{2}:\d{2}$" if walltime and re.match(pattern, walltime): self.set("time", str(walltime)) else: @@ -63,7 +64,7 @@ def set_walltime(self, walltime: str) -> None: def set_nodes(self, num_nodes: int) -> None: """Set the number of nodes for this batch job - + This sets ``--nodes``. :param num_nodes: number of nodes @@ -72,7 +73,7 @@ def set_nodes(self, num_nodes: int) -> None: def set_account(self, account: str) -> None: """Set the account for this batch job - + This sets ``--account``. :param account: account id @@ -81,7 +82,7 @@ def set_account(self, account: str) -> None: def set_partition(self, partition: str) -> None: """Set the partition for the batch job - + This sets ``--partition``. :param partition: partition name @@ -108,7 +109,7 @@ def set_cpus_per_task(self, cpus_per_task: int) -> None: def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job - + This sets ``--nodelist``. :param host_list: hosts to launch on @@ -142,7 +143,7 @@ def format_batch_args(self) -> t.List[str]: else: opts += ["=".join((prefix + opt, str(value)))] return opts - + def set(self, key: str, value: str | None) -> None: # Store custom arguments in the launcher_args - self._scheduler_args[key] = value \ No newline at end of file + self._scheduler_args[key] = value diff --git a/smartsim/settings/translators/batchArgBuilder.py b/smartsim/settings/translators/batchArgBuilder.py index 6105a1c73c..50426559d4 100644 --- a/smartsim/settings/translators/batchArgBuilder.py +++ b/smartsim/settings/translators/batchArgBuilder.py @@ -26,28 +26,28 @@ from __future__ import annotations -from abc import ABC, abstractmethod -import typing as t import copy +import typing as t +from abc import ABC, abstractmethod -from smartsim.log import get_logger +from smartsim.log import get_logger logger = get_logger(__name__) + class BatchArgBuilder(ABC): """Abstract base class that defines all generic scheduler argument methods that are not supported. It is the responsibility of child classes for each launcher to translate the input parameter to a properly formatted launcher argument. """ - + def __init__(self, scheduler_args: t.Dict[str, str | None] | None) -> None: self._scheduler_args = copy.deepcopy(scheduler_args) or {} @abstractmethod def scheduler_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" pass @abstractmethod @@ -99,4 +99,4 @@ def format_batch_args(self) -> t.List[str]: :return: batch arguments for Sbatch """ - pass \ No newline at end of file + pass diff --git a/smartsim/settings/translators/launch/alps.py b/smartsim/settings/translators/launch/alps.py index bf8eeea164..1c562344a5 100644 --- a/smartsim/settings/translators/launch/alps.py +++ b/smartsim/settings/translators/launch/alps.py @@ -25,30 +25,31 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import annotations -from ..launchArgBuilder import LaunchArgBuilder + import typing as t -from ...common import set_check_input, StringArgument + +from smartsim.log import get_logger + +from ...common import StringArgument, set_check_input from ...launchCommand import LauncherType -from smartsim.log import get_logger +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) -class AprunArgBuilder(LaunchArgBuilder): +class AprunArgBuilder(LaunchArgBuilder): def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def _reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ + """Return reserved launch arguments.""" return {"wdir"} def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Alps.value def set_cpus_per_task(self, cpus_per_task: int) -> None: @@ -58,7 +59,7 @@ def set_cpus_per_task(self, cpus_per_task: int) -> None: :param cpus_per_task: number of cpus to use per task """ - self.set("cpus-per-pe",str(cpus_per_task)) + self.set("cpus-per-pe", str(cpus_per_task)) def set_tasks(self, tasks: int) -> None: """Set the number of tasks for this job @@ -67,7 +68,7 @@ def set_tasks(self, tasks: int) -> None: :param tasks: number of tasks """ - self.set("pes",str(tasks)) + self.set("pes", str(tasks)) def set_tasks_per_node(self, tasks_per_node: int) -> None: """Set the number of tasks for this job @@ -76,13 +77,13 @@ def set_tasks_per_node(self, tasks_per_node: int) -> None: :param tasks_per_node: number of tasks per node """ - self.set("pes-per-node",str(tasks_per_node)) + self.set("pes-per-node", str(tasks_per_node)) def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify the hostlist for this job This sets ``--node-list`` - + :param host_list: hosts to launch on :raises TypeError: if not str or list of str """ @@ -101,11 +102,11 @@ def set_hostlist_from_file(self, file_path: str) -> None: :param file_path: Path to the hostlist file """ - self.set("node-list-file",file_path) - + self.set("node-list-file", file_path) + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: """Specify a list of hosts to exclude for launching this job - + This sets ``--exclude-node-list`` :param host_list: hosts to exclude @@ -156,7 +157,7 @@ def set_verbose_launch(self, verbose: bool) -> None: :param verbose: Whether the job should be run verbosely """ if verbose: - self.set("debug","7") + self.set("debug", "7") else: self._launch_args.pop("debug", None) @@ -168,11 +169,13 @@ def set_quiet_launch(self, quiet: bool) -> None: :param quiet: Whether the job should be run quietly """ if quiet: - self.set("quiet",None) + self.set("quiet", None) else: self._launch_args.pop("quiet", None) - def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: + def format_env_vars( + self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]] + ) -> t.Union[t.List[str], None]: """Format the environment variables for aprun :return: list of env vars @@ -183,7 +186,7 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> formatted += ["-e", name + "=" + str(value)] return formatted - def format_launch_args(self) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str], None]: """Return a list of ALPS formatted run arguments :return: list of ALPS arguments for these settings @@ -203,9 +206,8 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: return args def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._reserved_launch_args(): logger.warning( ( @@ -216,4 +218,4 @@ def set(self, key: str, value: str | None) -> None: return if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value \ No newline at end of file + self._launch_args[key] = value diff --git a/smartsim/settings/translators/launch/dragon.py b/smartsim/settings/translators/launch/dragon.py index 24b4fe0fd8..76c6e60a76 100644 --- a/smartsim/settings/translators/launch/dragon.py +++ b/smartsim/settings/translators/launch/dragon.py @@ -27,24 +27,25 @@ from __future__ import annotations import typing as t -from ..launchArgBuilder import LaunchArgBuilder + +from smartsim.log import get_logger + from ...common import StringArgument, set_check_input -from smartsim.log import get_logger -from ...launchCommand import LauncherType +from ...launchCommand import LauncherType +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) + class DragonArgBuilder(LaunchArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Dragon.value def set_nodes(self, nodes: int) -> None: @@ -52,19 +53,18 @@ def set_nodes(self, nodes: int) -> None: :param nodes: number of nodes to run with """ - self.set("nodes",str(nodes)) + self.set("nodes", str(nodes)) def set_tasks_per_node(self, tasks_per_node: int) -> None: """Set the number of tasks for this job :param tasks_per_node: number of tasks per node """ - self.set("tasks-per-node",str(tasks_per_node)) + self.set("tasks-per-node", str(tasks_per_node)) def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value \ No newline at end of file + self._launch_args[key] = value diff --git a/smartsim/settings/translators/launch/local.py b/smartsim/settings/translators/launch/local.py index 0ec3a7eb58..f664fb127a 100644 --- a/smartsim/settings/translators/launch/local.py +++ b/smartsim/settings/translators/launch/local.py @@ -27,27 +27,28 @@ from __future__ import annotations import typing as t -from ..launchArgBuilder import LaunchArgBuilder + from smartsim.log import get_logger -from ...common import StringArgument, set_check_input -from ...launchCommand import LauncherType + +from ...common import StringArgument, set_check_input +from ...launchCommand import LauncherType +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) + class LocalArgBuilder(LaunchArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Local.value - def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None]: + def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str], None]: """Build environment variable string :returns: formatted list of strings to export variables @@ -60,7 +61,7 @@ def format_env_vars(self, env_vars: StringArgument) -> t.Union[t.List[str],None] formatted.append(f"{key}={val}") return formatted - def format_launch_args(self) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str], None]: """Build launcher argument string :returns: formatted list of launcher arguments @@ -72,9 +73,8 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: return formatted def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value \ No newline at end of file + self._launch_args[key] = value diff --git a/smartsim/settings/translators/launch/lsf.py b/smartsim/settings/translators/launch/lsf.py index 51753a74dd..4d186e780e 100644 --- a/smartsim/settings/translators/launch/lsf.py +++ b/smartsim/settings/translators/launch/lsf.py @@ -27,29 +27,29 @@ from __future__ import annotations import typing as t -from ..launchArgBuilder import LaunchArgBuilder + +from smartsim.log import get_logger + from ...common import StringArgument, set_check_input -from smartsim.log import get_logger -from ...launchCommand import LauncherType +from ...launchCommand import LauncherType +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) + class JsrunArgBuilder(LaunchArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Lsf.value def _reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ + """Return reserved launch arguments.""" return {"chdir", "h", "stdio_stdout", "o", "stdio_stderr", "k"} def set_tasks(self, tasks: int) -> None: @@ -70,7 +70,9 @@ def set_binding(self, binding: str) -> None: """ self.set("bind", binding) - def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + def format_env_vars( + self, env_vars: t.Dict[str, t.Optional[str]] + ) -> t.Union[t.List[str], None]: """Format environment variables. Each variable needs to be passed with ``--env``. If a variable is set to ``None``, its value is propagated from the current environment. @@ -85,14 +87,14 @@ def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.L format_str += ["-E", f"{k}"] return format_str - def format_launch_args(self) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str], None]: """Return a list of LSF formatted run arguments :return: list of LSF arguments for these settings """ # args launcher uses args = [] - + for opt, value in self._launch_args.items(): short_arg = bool(len(str(opt)) == 1) prefix = "-" if short_arg else "--" @@ -106,9 +108,8 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: return args def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._reserved_launch_args(): logger.warning( ( @@ -119,4 +120,4 @@ def set(self, key: str, value: str | None) -> None: return if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value \ No newline at end of file + self._launch_args[key] = value diff --git a/smartsim/settings/translators/launch/mpi.py b/smartsim/settings/translators/launch/mpi.py index f83e9f4d51..e5c7194235 100644 --- a/smartsim/settings/translators/launch/mpi.py +++ b/smartsim/settings/translators/launch/mpi.py @@ -27,15 +27,17 @@ from __future__ import annotations import typing as t -from ..launchArgBuilder import LaunchArgBuilder + +from smartsim.log import get_logger + from ...common import set_check_input from ...launchCommand import LauncherType -from smartsim.log import get_logger +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) + class _BaseMPIArgBuilder(LaunchArgBuilder): - def __init__( self, launch_args: t.Dict[str, str | None] | None, @@ -43,12 +45,11 @@ def __init__( super().__init__(launch_args) def _reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ + """Return reserved launch arguments.""" return {"wd", "wdir"} def set_task_map(self, task_mapping: str) -> None: - """ Set ``mpirun`` task mapping + """Set ``mpirun`` task mapping this sets ``--map-by `` @@ -56,10 +57,10 @@ def set_task_map(self, task_mapping: str) -> None: :param task_mapping: task mapping """ - self.set("map-by",task_mapping) + self.set("map-by", task_mapping) def set_cpus_per_task(self, cpus_per_task: int) -> None: - """ Set the number of tasks for this job + """Set the number of tasks for this job This sets ``--cpus-per-proc`` for MPI compliant implementations @@ -84,26 +85,26 @@ def set_executable_broadcast(self, dest_path: str) -> None: "Using session directory instead" ) ) - self.set("preload-binary",dest_path) + self.set("preload-binary", dest_path) def set_cpu_binding_type(self, bind_type: str) -> None: - """ Specifies the cores to which MPI processes are bound + """Specifies the cores to which MPI processes are bound This sets ``--bind-to`` for MPI compliant implementations :param bind_type: binding type """ - self.set("bind-to",bind_type) + self.set("bind-to", bind_type) def set_tasks_per_node(self, tasks_per_node: int) -> None: - """ Set the number of tasks per node + """Set the number of tasks per node :param tasks_per_node: number of tasks to launch per node """ - self.set("npernode",str(tasks_per_node)) + self.set("npernode", str(tasks_per_node)) def set_tasks(self, tasks: int) -> None: - """ Set the number of tasks for this job + """Set the number of tasks for this job This sets ``-n`` for MPI compliant implementations @@ -112,7 +113,7 @@ def set_tasks(self, tasks: int) -> None: self.set("n", str(tasks)) def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - """ Set the hostlist for the ``mpirun`` command + """Set the hostlist for the ``mpirun`` command This sets ``--host`` @@ -127,24 +128,24 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: raise TypeError("host_list argument must be list of strings") self.set("host", ",".join(host_list)) - def set_hostlist_from_file(self, file_path: str) -> None: - """ Use the contents of a file to set the hostlist + def set_hostlist_from_file(self, file_path: str) -> None: + """Use the contents of a file to set the hostlist This sets ``--hostfile`` :param file_path: Path to the hostlist file """ - self.set("hostfile",file_path) + self.set("hostfile", file_path) def set_verbose_launch(self, verbose: bool) -> None: - """ Set the job to run in verbose mode + """Set the job to run in verbose mode This sets ``--verbose`` :param verbose: Whether the job should be run verbosely """ if verbose: - self.set("verbose",None) + self.set("verbose", None) else: self._launch_args.pop("verbose", None) @@ -155,22 +156,24 @@ def set_walltime(self, walltime: str) -> None: :param walltime: number like string of seconds that a job will run in secs """ - self.set("timeout",walltime) + self.set("timeout", walltime) def set_quiet_launch(self, quiet: bool) -> None: - """ Set the job to run in quiet mode + """Set the job to run in quiet mode This sets ``--quiet`` :param quiet: Whether the job should be run quietly """ if quiet: - self.set("quiet",None) + self.set("quiet", None) else: self._launch_args.pop("quiet", None) - def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: - """ Format the environment variables for mpirun + def format_env_vars( + self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]] + ) -> t.Union[t.List[str], None]: + """Format the environment variables for mpirun :return: list of env vars """ @@ -184,7 +187,7 @@ def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> else: formatted += [env_string, name] return formatted - + def format_launch_args(self) -> t.List[str]: """Return a list of MPI-standard formatted run arguments @@ -202,9 +205,8 @@ def format_launch_args(self) -> t.List[str]: return args def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._reserved_launch_args(): logger.warning( ( @@ -217,41 +219,38 @@ def set(self, key: str, value: str | None) -> None: logger.warning(f"Overwritting argument '{key}' with value '{value}'") self._launch_args[key] = value + class MpiArgBuilder(_BaseMPIArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Mpirun.value + class MpiexecArgBuilder(_BaseMPIArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Mpiexec.value + class OrteArgBuilder(_BaseMPIArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ - return LauncherType.Orterun.value \ No newline at end of file + """Get the string representation of the launcher""" + return LauncherType.Orterun.value diff --git a/smartsim/settings/translators/launch/pals.py b/smartsim/settings/translators/launch/pals.py index 8c46594dd6..98481e8c7e 100644 --- a/smartsim/settings/translators/launch/pals.py +++ b/smartsim/settings/translators/launch/pals.py @@ -27,46 +27,46 @@ from __future__ import annotations import typing as t -from ..launchArgBuilder import LaunchArgBuilder + +from smartsim.log import get_logger + from ...common import StringArgument, set_check_input -from smartsim.log import get_logger -from ...launchCommand import LauncherType +from ...launchCommand import LauncherType +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) + class PalsMpiexecArgBuilder(LaunchArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Pals.value - + def _reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ + """Return reserved launch arguments.""" return {"wdir", "wd"} def set_cpu_binding_type(self, bind_type: str) -> None: - """ Specifies the cores to which MPI processes are bound + """Specifies the cores to which MPI processes are bound This sets ``--bind-to`` for MPI compliant implementations :param bind_type: binding type """ - self.set("bind-to",bind_type) + self.set("bind-to", bind_type) def set_tasks(self, tasks: int) -> None: - """ Set the number of tasks + """Set the number of tasks :param tasks: number of total tasks to launch """ - self.set("np",str(tasks)) + self.set("np", str(tasks)) def set_executable_broadcast(self, dest_path: str) -> None: """Copy the specified executable(s) to remote machines @@ -78,16 +78,16 @@ def set_executable_broadcast(self, dest_path: str) -> None: self.set("transfer", dest_path) def set_tasks_per_node(self, tasks_per_node: int) -> None: - """ Set the number of tasks per node - + """Set the number of tasks per node + This sets ``--ppn`` :param tasks_per_node: number of tasks to launch per node """ - self.set("ppn",str(tasks_per_node)) + self.set("ppn", str(tasks_per_node)) def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - """ Set the hostlist for the PALS ``mpiexec`` command + """Set the hostlist for the PALS ``mpiexec`` command This sets ``hosts`` @@ -100,10 +100,12 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - self.set("hosts",",".join(host_list)) + self.set("hosts", ",".join(host_list)) - def format_env_vars(self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]]) -> t.Union[t.List[str],None]: - """ Format the environment variables for mpirun + def format_env_vars( + self, env_vars: t.Optional[t.Dict[str, t.Optional[str]]] + ) -> t.Union[t.List[str], None]: + """Format the environment variables for mpirun :return: list of env vars """ @@ -140,9 +142,8 @@ def format_launch_args(self) -> t.List[str]: return args def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._reserved_launch_args(): logger.warning( ( @@ -153,4 +154,4 @@ def set(self, key: str, value: str | None) -> None: return if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value \ No newline at end of file + self._launch_args[key] = value diff --git a/smartsim/settings/translators/launch/slurm.py b/smartsim/settings/translators/launch/slurm.py index 59c2c7de3b..6f6ad40692 100644 --- a/smartsim/settings/translators/launch/slurm.py +++ b/smartsim/settings/translators/launch/slurm.py @@ -26,36 +26,36 @@ from __future__ import annotations -import typing as t -import re import os -from ..launchArgBuilder import LaunchArgBuilder +import re +import typing as t + +from smartsim.log import get_logger + from ...common import IntegerArgument, StringArgument, set_check_input -from smartsim.log import get_logger -from ...launchCommand import LauncherType +from ...launchCommand import LauncherType +from ..launchArgBuilder import LaunchArgBuilder logger = get_logger(__name__) + class SlurmArgBuilder(LaunchArgBuilder): - def __init__( self, - launch_args: t.Dict[str, str | None] | None, + launch_args: t.Dict[str, str | None] | None, ) -> None: super().__init__(launch_args) - + def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" return LauncherType.Slurm.value def _reserved_launch_args(self) -> set[str]: - """ Return reserved launch arguments. - """ + """Return reserved launch arguments.""" return {"chdir", "D"} def set_nodes(self, nodes: int) -> None: - """ Set the number of nodes + """Set the number of nodes Effectively this is setting: ``srun --nodes `` @@ -63,9 +63,9 @@ def set_nodes(self, nodes: int) -> None: :return: launcher argument """ self.set("nodes", str(nodes)) - + def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: - """ Specify the hostlist for this job + """Specify the hostlist for this job This sets ``--nodelist`` @@ -81,7 +81,7 @@ def set_hostlist(self, host_list: t.Union[str, t.List[str]]) -> None: self.set("nodelist", ",".join(host_list)) def set_hostlist_from_file(self, file_path: str) -> None: - """ Use the contents of a file to set the node list + """Use the contents of a file to set the node list This sets ``--nodefile`` @@ -89,8 +89,8 @@ def set_hostlist_from_file(self, file_path: str) -> None: """ self.set("nodefile", file_path) - def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: - """ Specify a list of hosts to exclude for launching this job + def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: + """Specify a list of hosts to exclude for launching this job :param host_list: hosts to exclude :raises TypeError: if not str or list of str @@ -101,10 +101,10 @@ def set_excluded_hosts(self, host_list: t.Union[str, t.List[str]]) -> None: raise TypeError("host_list argument must be a list of strings") if not all(isinstance(host, str) for host in host_list): raise TypeError("host_list argument must be list of strings") - self.set("exclude",",".join(host_list)) + self.set("exclude", ",".join(host_list)) def set_cpus_per_task(self, cpus_per_task: int) -> None: - """ Set the number of cpus to use per task + """Set the number of cpus to use per task This sets ``--cpus-per-task`` @@ -113,25 +113,25 @@ def set_cpus_per_task(self, cpus_per_task: int) -> None: self.set("cpus-per-task", str(cpus_per_task)) def set_tasks(self, tasks: int) -> None: - """ Set the number of tasks for this job + """Set the number of tasks for this job This sets ``--ntasks`` :param tasks: number of tasks """ - self.set("ntasks",str(tasks)) - + self.set("ntasks", str(tasks)) + def set_tasks_per_node(self, tasks_per_node: int) -> None: - """ Set the number of tasks for this job + """Set the number of tasks for this job This sets ``--ntasks-per-node`` :param tasks_per_node: number of tasks per node """ - self.set("ntasks-per-node",str(tasks_per_node)) - - def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> None: - """ Bind by setting CPU masks on tasks + self.set("ntasks-per-node", str(tasks_per_node)) + + def set_cpu_bindings(self, bindings: t.Union[int, t.List[int]]) -> None: + """Bind by setting CPU masks on tasks This sets ``--cpu-bind`` using the ``map_cpu:`` option @@ -139,28 +139,28 @@ def set_cpu_bindings(self, bindings: t.Union[int,t.List[int]]) -> None: """ if isinstance(bindings, int): bindings = [bindings] - self.set("cpu_bind","map_cpu:" + ",".join(str(num) for num in bindings)) + self.set("cpu_bind", "map_cpu:" + ",".join(str(num) for num in bindings)) def set_memory_per_node(self, memory_per_node: int) -> None: - """ Specify the real memory required per node + """Specify the real memory required per node This sets ``--mem`` in megabytes :param memory_per_node: Amount of memory per node in megabytes """ - self.set("mem",f"{memory_per_node}M") + self.set("mem", f"{memory_per_node}M") def set_executable_broadcast(self, dest_path: str) -> None: - """ Copy executable file to allocated compute nodes + """Copy executable file to allocated compute nodes This sets ``--bcast`` :param dest_path: Path to copy an executable file """ - self.set("bcast",dest_path) + self.set("bcast", dest_path) def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: - """ Specify the node feature for this job + """Specify the node feature for this job This sets ``-C`` @@ -171,18 +171,18 @@ def set_node_feature(self, feature_list: t.Union[str, t.List[str]]) -> None: feature_list = [feature_list.strip()] elif not all(isinstance(feature, str) for feature in feature_list): raise TypeError("node_feature argument must be string or list of strings") - self.set("C",",".join(feature_list)) + self.set("C", ",".join(feature_list)) def set_walltime(self, walltime: str) -> None: - """ Set the walltime of the job + """Set the walltime of the job format = "HH:MM:SS" :param walltime: wall time """ - pattern = r'^\d{2}:\d{2}:\d{2}$' + pattern = r"^\d{2}:\d{2}:\d{2}$" if walltime and re.match(pattern, walltime): - self.set("time",str(walltime)) + self.set("time", str(walltime)) else: raise ValueError("Invalid walltime format. Please use 'HH:MM:SS' format.") @@ -204,17 +204,17 @@ def set_het_group(self, het_group: t.Iterable[int]) -> None: f"but max het group in allocation is {het_size-1}" ) raise ValueError(msg) - self.set("het-group",",".join(str(group) for group in het_group)) + self.set("het-group", ",".join(str(group) for group in het_group)) def set_verbose_launch(self, verbose: bool) -> None: - """ Set the job to run in verbose mode + """Set the job to run in verbose mode This sets ``--verbose`` :param verbose: Whether the job should be run verbosely """ if verbose: - self.set("verbose",None) + self.set("verbose", None) else: self._launch_args.pop("verbose", None) @@ -226,11 +226,11 @@ def set_quiet_launch(self, quiet: bool) -> None: :param quiet: Whether the job should be run quietly """ if quiet: - self.set("quiet",None) + self.set("quiet", None) else: self._launch_args.pop("quiet", None) - def format_launch_args(self) -> t.Union[t.List[str],None]: + def format_launch_args(self) -> t.Union[t.List[str], None]: """Return a list of slurm formatted launch arguments :return: list of slurm arguments for these settings @@ -247,8 +247,10 @@ def format_launch_args(self) -> t.Union[t.List[str],None]: else: formatted += ["=".join((prefix + key, str(value)))] return formatted - - def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + + def format_env_vars( + self, env_vars: t.Dict[str, t.Optional[str]] + ) -> t.Union[t.List[str], None]: """Build bash compatible environment variable string for Slurm :returns: the formatted string of environment variables @@ -256,7 +258,9 @@ def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.L self._check_env_vars(env_vars) return [f"{k}={v}" for k, v in env_vars.items() if "," not in str(v)] - def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.Tuple[str, t.List[str]],None]: + def format_comma_sep_env_vars( + self, env_vars: t.Dict[str, t.Optional[str]] + ) -> t.Union[t.Tuple[str, t.List[str]], None]: """Build environment variable string for Slurm Slurm takes exports in comma separated lists @@ -304,9 +308,8 @@ def _check_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> None: logger.warning(msg) def set(self, key: str, value: str | None) -> None: - """ Set the launch arguments - """ - set_check_input(key,value) + """Set the launch arguments""" + set_check_input(key, value) if key in self._reserved_launch_args(): logger.warning( ( @@ -317,4 +320,4 @@ def set(self, key: str, value: str | None) -> None: return if key in self._launch_args and key != self._launch_args[key]: logger.warning(f"Overwritting argument '{key}' with value '{value}'") - self._launch_args[key] = value \ No newline at end of file + self._launch_args[key] = value diff --git a/smartsim/settings/translators/launchArgBuilder.py b/smartsim/settings/translators/launchArgBuilder.py index eb85347485..f3560e140e 100644 --- a/smartsim/settings/translators/launchArgBuilder.py +++ b/smartsim/settings/translators/launchArgBuilder.py @@ -26,42 +26,45 @@ from __future__ import annotations -from abc import ABC, abstractmethod -import typing as t import copy +import typing as t +from abc import ABC, abstractmethod -from smartsim.log import get_logger +from smartsim.log import get_logger logger = get_logger(__name__) + class LaunchArgBuilder(ABC): """Abstract base class that defines all generic launcher argument methods that are not supported. It is the responsibility of child classes for each launcher to translate the input parameter to a properly formatted launcher argument. """ + def __init__(self, launch_args: t.Dict[str, str | None] | None) -> None: self._launch_args = copy.deepcopy(launch_args) or {} - + @abstractmethod def launcher_str(self) -> str: - """ Get the string representation of the launcher - """ + """Get the string representation of the launcher""" pass - + @abstractmethod def set(self, arg: str, val: str | None) -> None: - """ Set the launch arguments - """ + """Set the launch arguments""" pass - - def format_launch_args(self) -> t.Union[t.List[str],None]: - """ Build formatted launch arguments - """ - logger.warning(f"format_launcher_args() not supported for {self.launcher_str()}.") + + def format_launch_args(self) -> t.Union[t.List[str], None]: + """Build formatted launch arguments""" + logger.warning( + f"format_launcher_args() not supported for {self.launcher_str()}." + ) return None - def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.Tuple[str, t.List[str]],None]: + def format_comma_sep_env_vars( + self, env_vars: t.Dict[str, t.Optional[str]] + ) -> t.Union[t.Tuple[str, t.List[str]], None]: """Build environment variable string for Slurm Slurm takes exports in comma separated lists @@ -70,13 +73,17 @@ def format_comma_sep_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t :returns: the formatted string of environment variables """ - logger.warning(f"format_comma_sep_env_vars() not supported for {self.launcher_str()}.") + logger.warning( + f"format_comma_sep_env_vars() not supported for {self.launcher_str()}." + ) return None - def format_env_vars(self, env_vars: t.Dict[str, t.Optional[str]]) -> t.Union[t.List[str],None]: + def format_env_vars( + self, env_vars: t.Dict[str, t.Optional[str]] + ) -> t.Union[t.List[str], None]: """Build bash compatible environment variable string for Slurm :returns: the formatted string of environment variables """ logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") - return None \ No newline at end of file + return None diff --git a/smartsim/wlm/slurm.py b/smartsim/wlm/slurm.py index 63d6afe6ee..8e2b4f2b02 100644 --- a/smartsim/wlm/slurm.py +++ b/smartsim/wlm/slurm.py @@ -38,10 +38,13 @@ SSReservedKeywordError, ) from ..log import get_logger + + # from ..settings.slurmSettings import fmt_walltime # Mock function def fmt_walltime() -> None: ... + logger = get_logger(__name__) diff --git a/tests/temp_tests/steps_tests.py b/tests/temp_tests/steps_tests.py index 9b64e2630a..bd20607f32 100644 --- a/tests/temp_tests/steps_tests.py +++ b/tests/temp_tests/steps_tests.py @@ -1,8 +1,31 @@ -from smartsim._core.launcher.step import LocalStep, SrunStep, SbatchStep, QsubBatchStep, MpiexecStep, MpirunStep, OrterunStep, BsubBatchStep, JsrunStep, AprunStep -from smartsim.settings import RunSettings, SrunSettings, SbatchSettings, QsubBatchSettings, MpirunSettings, OrterunSettings, BsubBatchSettings, JsrunSettings, AprunSettings -from smartsim.entity import Model import pytest +from smartsim._core.launcher.step import ( + AprunStep, + BsubBatchStep, + JsrunStep, + LocalStep, + MpiexecStep, + MpirunStep, + OrterunStep, + QsubBatchStep, + SbatchStep, + SrunStep, +) +from smartsim.entity import Model +from smartsim.settings import ( + AprunSettings, + BsubBatchSettings, + JsrunSettings, + MpirunSettings, + OrterunSettings, + QsubBatchSettings, + RunSettings, + SbatchSettings, + SrunSettings, +) + + # Test creating a job step @pytest.mark.parametrize( "settings_type, step_type", @@ -26,15 +49,15 @@ RunSettings, LocalStep, id="local", - ) - ] + ), + ], ) -def test_instantiate_run_settings( - settings_type, step_type -): +def test_instantiate_run_settings(settings_type, step_type): run_settings = settings_type() run_settings.in_batch = True - model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings) + model = Model( + exe="echo", exe_args="hello", name="model_name", run_settings=run_settings + ) jobStep = step_type(entity=model, run_settings=model.run_settings) assert jobStep.run_settings == run_settings assert jobStep.entity == model @@ -42,6 +65,7 @@ def test_instantiate_run_settings( assert jobStep.cwd == model.path assert jobStep.step_settings == model.run_settings + # Test creating a mpi job step @pytest.mark.parametrize( "settings_type, step_type", @@ -56,14 +80,14 @@ def test_instantiate_run_settings( MpirunStep, id="mpirun", ), - ] + ], ) -def test_instantiate_mpi_run_settings( - settings_type, step_type -): +def test_instantiate_mpi_run_settings(settings_type, step_type): run_settings = settings_type(fail_if_missing_exec=False) run_settings.in_batch = True - model = Model(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings) + model = Model( + exe="echo", exe_args="hello", name="model_name", run_settings=run_settings + ) jobStep = step_type(entity=model, run_settings=model.run_settings) assert jobStep.run_settings == run_settings assert jobStep.entity == model @@ -71,6 +95,7 @@ def test_instantiate_mpi_run_settings( assert jobStep.cwd == model.path assert jobStep.step_settings == model.run_settings + # Test creating a batch job step @pytest.mark.parametrize( "settings_type, batch_settings_type, step_type", @@ -92,19 +117,23 @@ def test_instantiate_mpi_run_settings( QsubBatchSettings, QsubBatchStep, id="qsub", - ) - ] + ), + ], ) -def test_instantiate_batch_settings( - settings_type, batch_settings_type, step_type -): +def test_instantiate_batch_settings(settings_type, batch_settings_type, step_type): run_settings = settings_type() run_settings.in_batch = True batch_settings = batch_settings_type() - model = Application(exe="echo", exe_args="hello", name="model_name", run_settings=run_settings, batch_settings=batch_settings) + model = Application( + exe="echo", + exe_args="hello", + name="model_name", + run_settings=run_settings, + batch_settings=batch_settings, + ) jobStep = step_type(entity=model, batch_settings=model.batch_settings) assert jobStep.batch_settings == batch_settings assert jobStep.entity == model assert jobStep.entity_name == model.name assert jobStep.cwd == model.path - assert jobStep.step_settings == model.batch_settings \ No newline at end of file + assert jobStep.step_settings == model.batch_settings diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 507d154d68..8780357268 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -27,33 +27,42 @@ from smartsim._core.commands.command import Command from smartsim.settings.launchCommand import LauncherType + def test_command_init(): - cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) assert cmd.command == ["salloc", "-N", "1"] - assert cmd.launcher == LauncherType.SlurmLauncher + assert cmd.launcher == LauncherType.Slurm + def test_command_getitem(): - cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) get_value = cmd[0] assert get_value == "salloc" + def test_command_setitem(): - cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) cmd[0] = "srun" cmd[1] = "-n" assert cmd.command == ["srun", "-n", "1"] + def test_command_delitem(): - cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1", "--constraint", "P100"]) - del(cmd.command[3]) - del(cmd.command[3]) + cmd = Command( + launcher=LauncherType.Slurm, + command=["salloc", "-N", "1", "--constraint", "P100"], + ) + del cmd.command[3] + del cmd.command[3] assert cmd.command == ["salloc", "-N", "1"] + def test_command_len(): - cmd = Command(launcher=LauncherType.SlurmLauncher, command=["salloc", "-N", "1"]) + cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) assert len(cmd) is 3 + def test_command_insert(): - cmd = Command(launcher=LauncherType.SlurmLauncher, command=["-N", "1"]) + cmd = Command(launcher=LauncherType.Slurm, command=["-N", "1"]) cmd.insert(0, "salloc") - assert cmd.command == ["salloc", "-N", "1"] \ No newline at end of file + assert cmd.command == ["salloc", "-N", "1"] diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index 1779191de7..dcded85307 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -24,38 +24,44 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from smartsim._core.commands.commandList import CommandList from smartsim._core.commands.command import Command +from smartsim._core.commands.commandList import CommandList from smartsim.settings.launchCommand import LauncherType salloc_cmd = Command(launcher=LauncherType.Slurm, command=["salloc", "-N", "1"]) srun_cmd = Command(launcher=LauncherType.Slurm, command=["srun", "-n", "1"]) sacct_cmd = Command(launcher=LauncherType.Slurm, command=["sacct", "--user"]) + def test_command_init(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) - assert cmd_list.commands == [salloc_cmd,srun_cmd] + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) + assert cmd_list.commands == [salloc_cmd, srun_cmd] + def test_command_getitem(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) get_value = cmd_list[0] assert get_value == salloc_cmd + def test_command_setitem(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) cmd_list[0] = sacct_cmd - assert cmd_list.commands == [sacct_cmd,srun_cmd] + assert cmd_list.commands == [sacct_cmd, srun_cmd] + def test_command_delitem(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) - del(cmd_list.commands[0]) + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) + del cmd_list.commands[0] assert cmd_list.commands == [srun_cmd] + def test_command_len(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) assert len(cmd_list) is 2 + def test_command_insert(): - cmd_list = CommandList(commands=[salloc_cmd,srun_cmd]) + cmd_list = CommandList(commands=[salloc_cmd, srun_cmd]) cmd_list.insert(0, sacct_cmd) - assert cmd_list.commands == [sacct_cmd,salloc_cmd,srun_cmd] \ No newline at end of file + assert cmd_list.commands == [sacct_cmd, salloc_cmd, srun_cmd] diff --git a/tests/temp_tests/test_core/test_commands/test_launchCommands.py b/tests/temp_tests/test_core/test_commands/test_launchCommands.py index d76887e46a..65fb7c5a7c 100644 --- a/tests/temp_tests/test_core/test_commands/test_launchCommands.py +++ b/tests/temp_tests/test_core/test_commands/test_launchCommands.py @@ -1,5 +1,5 @@ -from smartsim._core.commands.commandList import CommandList from smartsim._core.commands.command import Command +from smartsim._core.commands.commandList import CommandList from smartsim._core.commands.launchCommands import LaunchCommands from smartsim.settings.launchCommand import LauncherType @@ -10,8 +10,13 @@ launch_command_list = CommandList(commands=[launch_cmd]) post_command_list = CommandList(commands=[post_cmd]) + def test_launchCommand_init(): - launch_cmd = LaunchCommands(prelaunch_commands=pre_commands_list,launch_commands=launch_command_list,postlaunch_commands=post_command_list) + launch_cmd = LaunchCommands( + prelaunch_commands=pre_commands_list, + launch_commands=launch_command_list, + postlaunch_commands=post_command_list, + ) assert launch_cmd.prelaunch_command == pre_commands_list assert launch_cmd.launch_command == launch_command_list assert launch_cmd.postlaunch_command == post_command_list diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 26937aff4f..2fcb85e2a5 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,63 +1,122 @@ import pytest + from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.alps import AprunArgBuilder from smartsim.settings.launchCommand import LauncherType +from smartsim.settings.translators.launch.alps import AprunArgBuilder + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) assert alpsLauncher.launch_args.launcher_str() == LauncherType.Alps.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_cpus_per_task", (4,),"4","cpus-per-pe",id="set_cpus_per_task"), - pytest.param("set_tasks", (4,),"4","pes",id="set_tasks"), - pytest.param("set_tasks_per_node", (4,),"4","pes-per-node",id="set_tasks_per_node"), - pytest.param("set_hostlist", ("host_A",),"host_A","node-list",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","node-list",id="set_hostlist_list[str]"), - pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","node-list-file",id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude-node-list",id="set_excluded_hosts_str"), - pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude-node-list",id="set_excluded_hosts_list[str]"), - pytest.param("set_cpu_bindings", (4,),"4","cpu-binding",id="set_cpu_bindings"), - pytest.param("set_cpu_bindings", ([4,4],),"4,4","cpu-binding",id="set_cpu_bindings_list[str]"), - pytest.param("set_memory_per_node", (8000,),"8000","memory-per-pe",id="set_memory_per_node"), - pytest.param("set_walltime", ("10:00:00",),"10:00:00","cpu-time-limit",id="set_walltime"), - pytest.param("set_verbose_launch", (True,),"7","debug",id="set_verbose_launch"), - pytest.param("set_quiet_launch", (True,),None,"quiet",id="set_quiet_launch"), + pytest.param( + "set_cpus_per_task", (4,), "4", "cpus-per-pe", id="set_cpus_per_task" + ), + pytest.param("set_tasks", (4,), "4", "pes", id="set_tasks"), + pytest.param( + "set_tasks_per_node", (4,), "4", "pes-per-node", id="set_tasks_per_node" + ), + pytest.param( + "set_hostlist", ("host_A",), "host_A", "node-list", id="set_hostlist_str" + ), + pytest.param( + "set_hostlist", + (["host_A", "host_B"],), + "host_A,host_B", + "node-list", + id="set_hostlist_list[str]", + ), + pytest.param( + "set_hostlist_from_file", + ("./path/to/hostfile",), + "./path/to/hostfile", + "node-list-file", + id="set_hostlist_from_file", + ), + pytest.param( + "set_excluded_hosts", + ("host_A",), + "host_A", + "exclude-node-list", + id="set_excluded_hosts_str", + ), + pytest.param( + "set_excluded_hosts", + (["host_A", "host_B"],), + "host_A,host_B", + "exclude-node-list", + id="set_excluded_hosts_list[str]", + ), + pytest.param( + "set_cpu_bindings", (4,), "4", "cpu-binding", id="set_cpu_bindings" + ), + pytest.param( + "set_cpu_bindings", + ([4, 4],), + "4,4", + "cpu-binding", + id="set_cpu_bindings_list[str]", + ), + pytest.param( + "set_memory_per_node", + (8000,), + "8000", + "memory-per-pe", + id="set_memory_per_node", + ), + pytest.param( + "set_walltime", + ("10:00:00",), + "10:00:00", + "cpu-time-limit", + id="set_walltime", + ), + pytest.param( + "set_verbose_launch", (True,), "7", "debug", id="set_verbose_launch" + ), + pytest.param("set_quiet_launch", (True,), None, "quiet", id="set_quiet_launch"), ], ) def test_alps_class_methods(function, value, flag, result): alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) - assert isinstance(alpsLauncher._arg_builder,AprunArgBuilder) + assert isinstance(alpsLauncher._arg_builder, AprunArgBuilder) getattr(alpsLauncher.launch_args, function)(*value) assert alpsLauncher.launch_args._launch_args[flag] == result + def test_set_verbose_launch(): alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) - assert isinstance(alpsLauncher._arg_builder,AprunArgBuilder) + assert isinstance(alpsLauncher._arg_builder, AprunArgBuilder) alpsLauncher.launch_args.set_verbose_launch(True) - assert alpsLauncher.launch_args._launch_args == {'debug': "7"} + assert alpsLauncher.launch_args._launch_args == {"debug": "7"} alpsLauncher.launch_args.set_verbose_launch(False) assert alpsLauncher.launch_args._launch_args == {} + def test_set_quiet_launch(): aprunLauncher = LaunchSettings(launcher=LauncherType.Alps) - assert isinstance(aprunLauncher._arg_builder,AprunArgBuilder) + assert isinstance(aprunLauncher._arg_builder, AprunArgBuilder) aprunLauncher.launch_args.set_quiet_launch(True) - assert aprunLauncher.launch_args._launch_args == {'quiet': None} + assert aprunLauncher.launch_args._launch_args == {"quiet": None} aprunLauncher.launch_args.set_quiet_launch(False) assert aprunLauncher.launch_args._launch_args == {} - + + def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose"} aprunLauncher = LaunchSettings(launcher=LauncherType.Alps, env_vars=env_vars) - assert isinstance(aprunLauncher._arg_builder,AprunArgBuilder) + assert isinstance(aprunLauncher._arg_builder, AprunArgBuilder) aprunLauncher.update_env({"OMP_NUM_THREADS": "10"}) formatted = aprunLauncher.format_env_vars() result = ["-e", "OMP_NUM_THREADS=10", "-e", "LOGGING=verbose"] assert formatted == result + def test_aprun_settings(): aprunLauncher = LaunchSettings(launcher=LauncherType.Alps) aprunLauncher.launch_args.set_cpus_per_task(2) @@ -67,22 +126,24 @@ def test_aprun_settings(): result = ["--cpus-per-pe=2", "--pes=100", "--pes-per-node=20"] assert formatted == result + def test_invalid_hostlist_format(): """Test invalid hostlist formats""" alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) with pytest.raises(TypeError): - alpsLauncher.launch_args.set_hostlist(["test",5]) + alpsLauncher.launch_args.set_hostlist(["test", 5]) with pytest.raises(TypeError): alpsLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): alpsLauncher.launch_args.set_hostlist(5) + def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" alpsLauncher = LaunchSettings(launcher=LauncherType.Alps) with pytest.raises(TypeError): - alpsLauncher.launch_args.set_excluded_hosts(["test",5]) + alpsLauncher.launch_args.set_excluded_hosts(["test", 5]) with pytest.raises(TypeError): alpsLauncher.launch_args.set_excluded_hosts([5]) with pytest.raises(TypeError): - alpsLauncher.launch_args.set_excluded_hosts(5) \ No newline at end of file + alpsLauncher.launch_args.set_excluded_hosts(5) diff --git a/tests/temp_tests/test_settings/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py index 323268a2cb..ab3490a765 100644 --- a/tests/temp_tests/test_settings/test_batchSettings.py +++ b/tests/temp_tests/test_settings/test_batchSettings.py @@ -1,6 +1,8 @@ +import pytest + from smartsim.settings import BatchSettings from smartsim.settings.batchCommand import SchedulerType -import pytest + @pytest.mark.parametrize( "scheduler_enum", @@ -11,26 +13,37 @@ ], ) def test_create_scheduler_settings(scheduler_enum): - bs_str = BatchSettings(batch_scheduler=scheduler_enum.value, scheduler_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + bs_str = BatchSettings( + batch_scheduler=scheduler_enum.value, + scheduler_args={"launch": "var"}, + env_vars={"ENV": "VAR"}, + ) assert bs_str._batch_scheduler == scheduler_enum # TODO need to test scheduler_args - assert bs_str._env_vars == {"ENV":"VAR"} - - bs_enum = BatchSettings(batch_scheduler=scheduler_enum, scheduler_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + assert bs_str._env_vars == {"ENV": "VAR"} + + bs_enum = BatchSettings( + batch_scheduler=scheduler_enum, + scheduler_args={"launch": "var"}, + env_vars={"ENV": "VAR"}, + ) assert bs_enum._batch_scheduler == scheduler_enum # TODO need to test scheduler_args - assert bs_enum._env_vars == {"ENV":"VAR"} + assert bs_enum._env_vars == {"ENV": "VAR"} + def test_launcher_property(): bs = BatchSettings(batch_scheduler="slurm") assert bs.batch_scheduler == "slurm" + def test_env_vars_property(): - bs = BatchSettings(batch_scheduler="slurm", env_vars={"ENV":"VAR"}) - assert bs.env_vars == {"ENV":"VAR"} + bs = BatchSettings(batch_scheduler="slurm", env_vars={"ENV": "VAR"}) + assert bs.env_vars == {"ENV": "VAR"} + def test_env_vars_property_deep_copy(): - bs = BatchSettings(batch_scheduler="slurm", env_vars={"ENV":"VAR"}) + bs = BatchSettings(batch_scheduler="slurm", env_vars={"ENV": "VAR"}) copy_env_vars = bs.env_vars - copy_env_vars.update({"test":"no_update"}) - assert bs.env_vars == {"ENV":"VAR"} \ No newline at end of file + copy_env_vars.update({"test": "no_update"}) + assert bs.env_vars == {"ENV": "VAR"} diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index f07c1ce5e1..994e90f5cc 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,22 +1,27 @@ +import pytest + from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.dragon import DragonArgBuilder from smartsim.settings.launchCommand import LauncherType -import pytest +from smartsim.settings.translators.launch.dragon import DragonArgBuilder + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" ls = LaunchSettings(launcher=LauncherType.Dragon) assert ls.launch_args.launcher_str() == LauncherType.Dragon.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), - pytest.param("set_tasks_per_node", (2,),"2","tasks-per-node",id="set_tasks_per_node"), + pytest.param("set_nodes", (2,), "2", "nodes", id="set_nodes"), + pytest.param( + "set_tasks_per_node", (2,), "2", "tasks-per-node", id="set_tasks_per_node" + ), ], ) def test_dragon_class_methods(function, value, flag, result): dragonLauncher = LaunchSettings(launcher=LauncherType.Dragon) - assert isinstance(dragonLauncher._arg_builder,DragonArgBuilder) + assert isinstance(dragonLauncher._arg_builder, DragonArgBuilder) getattr(dragonLauncher.launch_args, function)(*value) - assert dragonLauncher.launch_args._launch_args[flag] == result \ No newline at end of file + assert dragonLauncher.launch_args._launch_args[flag] == result diff --git a/tests/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py index f9e569bd0f..ff5e2dd67f 100644 --- a/tests/temp_tests/test_settings/test_launchSettings.py +++ b/tests/temp_tests/test_settings/test_launchSettings.py @@ -1,62 +1,76 @@ +import logging + +import pytest + from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -import pytest -import logging + @pytest.mark.parametrize( "launch_enum", [ - pytest.param(LauncherType.Slurm,id="slurm"), - pytest.param(LauncherType.Dragon,id="dragon"), - pytest.param(LauncherType.Pals,id="pals"), - pytest.param(LauncherType.Alps,id="alps"), - pytest.param(LauncherType.Local,id="local"), - pytest.param(LauncherType.Mpiexec,id="mpiexec"), - pytest.param(LauncherType.Mpirun,id="mpirun"), - pytest.param(LauncherType.Orterun,id="orterun"), - pytest.param(LauncherType.Lsf,id="lsf"), + pytest.param(LauncherType.Slurm, id="slurm"), + pytest.param(LauncherType.Dragon, id="dragon"), + pytest.param(LauncherType.Pals, id="pals"), + pytest.param(LauncherType.Alps, id="alps"), + pytest.param(LauncherType.Local, id="local"), + pytest.param(LauncherType.Mpiexec, id="mpiexec"), + pytest.param(LauncherType.Mpirun, id="mpirun"), + pytest.param(LauncherType.Orterun, id="orterun"), + pytest.param(LauncherType.Lsf, id="lsf"), ], ) def test_create_launch_settings(launch_enum): - ls_str = LaunchSettings(launcher=launch_enum.value, launch_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + ls_str = LaunchSettings( + launcher=launch_enum.value, + launch_args={"launch": "var"}, + env_vars={"ENV": "VAR"}, + ) assert ls_str._launcher == launch_enum # TODO need to test launch_args - assert ls_str._env_vars == {"ENV":"VAR"} - - ls_enum = LaunchSettings(launcher=launch_enum, launch_args={"launch":"var"}, env_vars={"ENV":"VAR"}) + assert ls_str._env_vars == {"ENV": "VAR"} + + ls_enum = LaunchSettings( + launcher=launch_enum, launch_args={"launch": "var"}, env_vars={"ENV": "VAR"} + ) assert ls_enum._launcher == launch_enum # TODO need to test launch_args - assert ls_enum._env_vars == {"ENV":"VAR"} + assert ls_enum._env_vars == {"ENV": "VAR"} + def test_launcher_property(): ls = LaunchSettings(launcher="local") assert ls.launcher == "local" + def test_env_vars_property(): - ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) - assert ls.env_vars == {"ENV":"VAR"} + ls = LaunchSettings(launcher="local", env_vars={"ENV": "VAR"}) + assert ls.env_vars == {"ENV": "VAR"} + def test_env_vars_property_deep_copy(): - ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) + ls = LaunchSettings(launcher="local", env_vars={"ENV": "VAR"}) copy_env_vars = ls.env_vars - copy_env_vars.update({"test":"no_update"}) - assert ls.env_vars == {"ENV":"VAR"} + copy_env_vars.update({"test": "no_update"}) + assert ls.env_vars == {"ENV": "VAR"} + def test_update_env_vars(): - ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) - ls.update_env({"test":"no_update"}) - assert ls.env_vars == {"ENV":"VAR","test":"no_update"} - + ls = LaunchSettings(launcher="local", env_vars={"ENV": "VAR"}) + ls.update_env({"test": "no_update"}) + assert ls.env_vars == {"ENV": "VAR", "test": "no_update"} + + def test_update_env_vars_errors(): - ls = LaunchSettings(launcher="local", env_vars={"ENV":"VAR"}) + ls = LaunchSettings(launcher="local", env_vars={"ENV": "VAR"}) with pytest.raises(TypeError): - ls.update_env({"test":1}) + ls.update_env({"test": 1}) with pytest.raises(TypeError): - ls.update_env({1:"test"}) + ls.update_env({1: "test"}) with pytest.raises(TypeError): - ls.update_env({1:1}) + ls.update_env({1: 1}) with pytest.raises(TypeError): # Make sure the first key and value do not assign # and that the function is atomic - ls.update_env({"test":"test","test":1}) - assert ls.env_vars == {"ENV":"VAR"} \ No newline at end of file + ls.update_env({"test": "test", "test": 1}) + assert ls.env_vars == {"ENV": "VAR"} diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 477815d1f2..b0ec3328b4 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,13 +1,16 @@ +import pytest + from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.local import LocalArgBuilder from smartsim.settings.launchCommand import LauncherType -import pytest +from smartsim.settings.translators.launch.local import LocalArgBuilder + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" ls = LaunchSettings(launcher=LauncherType.Local) assert ls.launch_args.launcher_str() == LauncherType.Local.value + # TODO complete after launch args retrieval def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization @@ -19,7 +22,9 @@ def test_launch_args_input_mutation(): key1: val1, key2: val2, } - localLauncher = LaunchSettings(launcher=LauncherType.Local, launch_args=default_launcher_args) + localLauncher = LaunchSettings( + launcher=LauncherType.Local, launch_args=default_launcher_args + ) # Confirm initial values are set assert localLauncher.launch_args._launch_args[key0] == val0 @@ -33,6 +38,7 @@ def test_launch_args_input_mutation(): # Confirm previously created run settings are not changed assert localLauncher.launch_args._launch_args[key2] == val2 + @pytest.mark.parametrize( "env_vars", [ @@ -49,11 +55,13 @@ def test_update_env(env_vars): assert len(localLauncher.env_vars) == len(env_vars.keys()) + def test_format_launch_args(): localLauncher = LaunchSettings(launcher=LauncherType.Local, launch_args={"-np": 2}) launch_args = localLauncher.format_launch_args() assert launch_args == ["-np", "2"] + @pytest.mark.parametrize( "env_vars", [ @@ -70,6 +78,7 @@ def test_update_env_null_valued(env_vars): localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=orig_env) localLauncher.update_env(env_vars) + @pytest.mark.parametrize( "env_vars", [ @@ -91,13 +100,14 @@ def test_update_env_initialized(env_vars): assert len(localLauncher.env_vars) == len(combined_keys) assert {k for k in localLauncher.env_vars.keys()} == combined_keys + def test_format_env_vars(): - env_vars={ - "A": "a", - "B": None, - "C": "", - "D": "12", - } + env_vars = { + "A": "a", + "B": None, + "C": "", + "D": "12", + } localLauncher = LaunchSettings(launcher=LauncherType.Local, env_vars=env_vars) assert isinstance(localLauncher._arg_builder, LocalArgBuilder) - assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] \ No newline at end of file + assert localLauncher.format_env_vars() == ["A=a", "B=", "C=", "D=12"] diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index c9260ec58a..f87b177ece 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,33 +1,40 @@ +import pytest + from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.lsf import JsrunArgBuilder from smartsim.settings.launchCommand import LauncherType -import pytest +from smartsim.settings.translators.launch.lsf import JsrunArgBuilder + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" ls = LaunchSettings(launcher=LauncherType.Lsf) assert ls.launch_args.launcher_str() == LauncherType.Lsf.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_tasks", (2,),"2","np",id="set_tasks"), - pytest.param("set_binding", ("packed:21",),"packed:21","bind",id="set_binding"), + pytest.param("set_tasks", (2,), "2", "np", id="set_tasks"), + pytest.param( + "set_binding", ("packed:21",), "packed:21", "bind", id="set_binding" + ), ], ) def test_lsf_class_methods(function, value, flag, result): lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf) - assert isinstance(lsfLauncher._arg_builder,JsrunArgBuilder) + assert isinstance(lsfLauncher._arg_builder, JsrunArgBuilder) getattr(lsfLauncher.launch_args, function)(*value) assert lsfLauncher.launch_args._launch_args[flag] == result + def test_format_env_vars(): env_vars = {"OMP_NUM_THREADS": None, "LOGGING": "verbose"} lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf, env_vars=env_vars) - assert isinstance(lsfLauncher._arg_builder,JsrunArgBuilder) + assert isinstance(lsfLauncher._arg_builder, JsrunArgBuilder) formatted = lsfLauncher.format_env_vars() assert formatted == ["-E", "OMP_NUM_THREADS", "-E", "LOGGING=verbose"] + def test_launch_args(): """Test the possible user overrides through run_args""" launch_args = { @@ -38,7 +45,7 @@ def test_launch_args(): "np": 100, } lsfLauncher = LaunchSettings(launcher=LauncherType.Lsf, launch_args=launch_args) - assert isinstance(lsfLauncher._arg_builder,JsrunArgBuilder) + assert isinstance(lsfLauncher._arg_builder, JsrunArgBuilder) formatted = lsfLauncher.format_launch_args() result = [ "--latency_priority=gpu-gpu", @@ -48,4 +55,4 @@ def test_launch_args(): "--nrs=10", "--np=100", ] - assert formatted == result \ No newline at end of file + assert formatted == result diff --git a/tests/temp_tests/test_settings/test_lsfScheduler.py b/tests/temp_tests/test_settings/test_lsfScheduler.py index 4cc21df1a6..5c93d8978c 100644 --- a/tests/temp_tests/test_settings/test_lsfScheduler.py +++ b/tests/temp_tests/test_settings/test_lsfScheduler.py @@ -1,24 +1,35 @@ -from smartsim.settings import BatchSettings import pytest + +from smartsim.settings import BatchSettings from smartsim.settings.batchCommand import SchedulerType + def test_scheduler_str(): """Ensure scheduler_str returns appropriate value""" bs = BatchSettings(batch_scheduler=SchedulerType.Lsf) assert bs.scheduler_args.scheduler_str() == SchedulerType.Lsf.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),"2","nnodes",id="set_nodes"), - pytest.param("set_walltime", ("10:00:00",),"10:00","W",id="set_walltime"), - pytest.param("set_hostlist", ("host_A",),""'"host_A"'"","m",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),""'"host_A host_B"'"","m",id="set_hostlist_list[str]"), - pytest.param("set_smts", (1,),"1","alloc_flags",id="set_smts"), - pytest.param("set_project", ("project",),"project","P",id="set_project"), - pytest.param("set_account", ("project",),"project","P",id="set_account"), - pytest.param("set_tasks", (2,),"2","n",id="set_tasks"), - pytest.param("set_queue", ("queue",),"queue","q",id="set_queue"), + pytest.param("set_nodes", (2,), "2", "nnodes", id="set_nodes"), + pytest.param("set_walltime", ("10:00:00",), "10:00", "W", id="set_walltime"), + pytest.param( + "set_hostlist", ("host_A",), "" '"host_A"' "", "m", id="set_hostlist_str" + ), + pytest.param( + "set_hostlist", + (["host_A", "host_B"],), + "" '"host_A host_B"' "", + "m", + id="set_hostlist_list[str]", + ), + pytest.param("set_smts", (1,), "1", "alloc_flags", id="set_smts"), + pytest.param("set_project", ("project",), "project", "P", id="set_project"), + pytest.param("set_account", ("project",), "project", "P", id="set_account"), + pytest.param("set_tasks", (2,), "2", "n", id="set_tasks"), + pytest.param("set_queue", ("queue",), "queue", "q", id="set_queue"), ], ) def test_update_env_initialized(function, value, flag, result): @@ -26,11 +37,14 @@ def test_update_env_initialized(function, value, flag, result): getattr(lsfScheduler.scheduler_args, function)(*value) assert lsfScheduler.scheduler_args._scheduler_args[flag] == result + def test_create_bsub(): batch_args = {"core_isolation": None} - lsfScheduler = BatchSettings(batch_scheduler=SchedulerType.Lsf, scheduler_args=batch_args) + lsfScheduler = BatchSettings( + batch_scheduler=SchedulerType.Lsf, scheduler_args=batch_args + ) lsfScheduler.scheduler_args.set_nodes(1) lsfScheduler.scheduler_args.set_walltime("10:10:10") lsfScheduler.scheduler_args.set_queue("default") args = lsfScheduler.format_batch_args() - assert args == ["-core_isolation", "-nnodes", "1", "-W", "10:10", "-q", "default"] \ No newline at end of file + assert args == ["-core_isolation", "-nnodes", "1", "-W", "10:10", "-q", "default"] diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 96e97aadcc..9de34cca50 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -1,8 +1,15 @@ -from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder -import pytest import itertools + +import pytest + +from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType +from smartsim.settings.translators.launch.mpi import ( + MpiArgBuilder, + MpiexecArgBuilder, + OrteArgBuilder, +) + @pytest.mark.parametrize( "launcher", @@ -17,34 +24,99 @@ def test_launcher_str(launcher): ls = LaunchSettings(launcher=launcher) assert ls.launch_args.launcher_str() == launcher.value + @pytest.mark.parametrize( "l,function,value,result,flag", [ - # Use OpenMPI style settigs for all launchers - *itertools.chain.from_iterable( - ( + # Use OpenMPI style settigs for all launchers + *itertools.chain.from_iterable( ( - pytest.param(l, "set_walltime", ("100",),"100","timeout",id="set_walltime"), - pytest.param(l, "set_task_map", ("taskmap",),"taskmap","map-by",id="set_task_map"), - pytest.param(l, "set_cpus_per_task", (2,),"2","cpus-per-proc",id="set_cpus_per_task"), - pytest.param(l, "set_cpu_binding_type", ("4",),"4","bind-to",id="set_cpu_binding_type"), - pytest.param(l, "set_tasks_per_node", (4,),"4","npernode",id="set_tasks_per_node"), - pytest.param(l, "set_tasks", (4,),"4","n",id="set_tasks"), - pytest.param(l, "set_executable_broadcast", ("broadcast",),"broadcast","preload-binary",id="set_executable_broadcast"), - pytest.param(l, "set_hostlist", ("host_A",),"host_A","host",id="set_hostlist_str"), - pytest.param(l, "set_hostlist", (["host_A","host_B"],),"host_A,host_B","host",id="set_hostlist_list[str]"), - pytest.param(l, "set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","hostfile",id="set_hostlist_from_file"), + ( + pytest.param( + l, "set_walltime", ("100",), "100", "timeout", id="set_walltime" + ), + pytest.param( + l, + "set_task_map", + ("taskmap",), + "taskmap", + "map-by", + id="set_task_map", + ), + pytest.param( + l, + "set_cpus_per_task", + (2,), + "2", + "cpus-per-proc", + id="set_cpus_per_task", + ), + pytest.param( + l, + "set_cpu_binding_type", + ("4",), + "4", + "bind-to", + id="set_cpu_binding_type", + ), + pytest.param( + l, + "set_tasks_per_node", + (4,), + "4", + "npernode", + id="set_tasks_per_node", + ), + pytest.param(l, "set_tasks", (4,), "4", "n", id="set_tasks"), + pytest.param( + l, + "set_executable_broadcast", + ("broadcast",), + "broadcast", + "preload-binary", + id="set_executable_broadcast", + ), + pytest.param( + l, + "set_hostlist", + ("host_A",), + "host_A", + "host", + id="set_hostlist_str", + ), + pytest.param( + l, + "set_hostlist", + (["host_A", "host_B"],), + "host_A,host_B", + "host", + id="set_hostlist_list[str]", + ), + pytest.param( + l, + "set_hostlist_from_file", + ("./path/to/hostfile",), + "./path/to/hostfile", + "hostfile", + id="set_hostlist_from_file", + ), + ) + for l in ( + [LauncherType.Mpirun, MpiArgBuilder], + [LauncherType.Mpiexec, MpiexecArgBuilder], + [LauncherType.Orterun, OrteArgBuilder], + ) + ) ) - for l in ([LauncherType.Mpirun, MpiArgBuilder], [LauncherType.Mpiexec, MpiexecArgBuilder], [LauncherType.Orterun, OrteArgBuilder]) - )) ], ) -def test_mpi_class_methods(l,function, value, flag, result): +def test_mpi_class_methods(l, function, value, flag, result): mpiSettings = LaunchSettings(launcher=l[0]) - assert isinstance(mpiSettings._arg_builder,l[1]) + assert isinstance(mpiSettings._arg_builder, l[1]) getattr(mpiSettings.launch_args, function)(*value) assert mpiSettings.launch_args._launch_args[flag] == result + @pytest.mark.parametrize( "launcher", [ @@ -65,6 +137,7 @@ def test_format_env_vars(launcher): ] assert formatted == result + @pytest.mark.parametrize( "launcher", [ @@ -82,6 +155,7 @@ def test_format_launcher_args(launcher): result = ["--cpus-per-proc", "1", "--n", "2", "--host", "node005,node006"] assert formatted == result + @pytest.mark.parametrize( "launcher", [ @@ -93,10 +167,11 @@ def test_format_launcher_args(launcher): def test_set_verbose_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) mpiSettings.launch_args.set_verbose_launch(True) - assert mpiSettings.launch_args._launch_args == {'verbose': None} + assert mpiSettings.launch_args._launch_args == {"verbose": None} mpiSettings.launch_args.set_verbose_launch(False) assert mpiSettings.launch_args._launch_args == {} + @pytest.mark.parametrize( "launcher", [ @@ -108,10 +183,11 @@ def test_set_verbose_launch(launcher): def test_set_quiet_launch(launcher): mpiSettings = LaunchSettings(launcher=launcher) mpiSettings.launch_args.set_quiet_launch(True) - assert mpiSettings.launch_args._launch_args == {'quiet': None} + assert mpiSettings.launch_args._launch_args == {"quiet": None} mpiSettings.launch_args.set_quiet_launch(False) assert mpiSettings.launch_args._launch_args == {} + @pytest.mark.parametrize( "launcher", [ @@ -124,8 +200,8 @@ def test_invalid_hostlist_format(launcher): """Test invalid hostlist formats""" mpiSettings = LaunchSettings(launcher=launcher) with pytest.raises(TypeError): - mpiSettings.launch_args.set_hostlist(["test",5]) + mpiSettings.launch_args.set_hostlist(["test", 5]) with pytest.raises(TypeError): mpiSettings.launch_args.set_hostlist([5]) with pytest.raises(TypeError): - mpiSettings.launch_args.set_hostlist(5) \ No newline at end of file + mpiSettings.launch_args.set_hostlist(5) diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 15e28a4780..5fbbccab77 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -1,31 +1,55 @@ +import pytest + from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.pals import PalsMpiexecArgBuilder from smartsim.settings.launchCommand import LauncherType -import pytest +from smartsim.settings.translators.launch.pals import PalsMpiexecArgBuilder + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" ls = LaunchSettings(launcher=LauncherType.Pals) assert ls.launch_args.launcher_str() == LauncherType.Pals.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_cpu_binding_type", ("bind",),"bind","bind-to",id="set_cpu_binding_type"), - pytest.param("set_tasks", (2,),"2","np",id="set_tasks"), - pytest.param("set_tasks_per_node", (2,),"2","ppn",id="set_tasks_per_node"), - pytest.param("set_hostlist", ("host_A",),"host_A","hosts",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hosts",id="set_hostlist_list[str]"), - pytest.param("set_executable_broadcast", ("broadcast",),"broadcast","transfer",id="set_executable_broadcast"), + pytest.param( + "set_cpu_binding_type", + ("bind",), + "bind", + "bind-to", + id="set_cpu_binding_type", + ), + pytest.param("set_tasks", (2,), "2", "np", id="set_tasks"), + pytest.param("set_tasks_per_node", (2,), "2", "ppn", id="set_tasks_per_node"), + pytest.param( + "set_hostlist", ("host_A",), "host_A", "hosts", id="set_hostlist_str" + ), + pytest.param( + "set_hostlist", + (["host_A", "host_B"],), + "host_A,host_B", + "hosts", + id="set_hostlist_list[str]", + ), + pytest.param( + "set_executable_broadcast", + ("broadcast",), + "broadcast", + "transfer", + id="set_executable_broadcast", + ), ], ) def test_pals_class_methods(function, value, flag, result): palsLauncher = LaunchSettings(launcher=LauncherType.Pals) - assert isinstance(palsLauncher.launch_args,PalsMpiexecArgBuilder) + assert isinstance(palsLauncher.launch_args, PalsMpiexecArgBuilder) getattr(palsLauncher.launch_args, function)(*value) assert palsLauncher.launch_args._launch_args[flag] == result assert palsLauncher.format_launch_args() == ["--" + flag, str(result)] + def test_format_env_vars(): env_vars = {"FOO_VERSION": "3.14", "PATH": None, "LD_LIBRARY_PATH": None} palsLauncher = LaunchSettings(launcher=LauncherType.Pals, env_vars=env_vars) @@ -33,12 +57,13 @@ def test_format_env_vars(): expected = "--env FOO_VERSION=3.14 --envlist PATH,LD_LIBRARY_PATH" assert formatted == expected + def test_invalid_hostlist_format(): """Test invalid hostlist formats""" palsLauncher = LaunchSettings(launcher=LauncherType.Pals) with pytest.raises(TypeError): - palsLauncher.launch_args.set_hostlist(["test",5]) + palsLauncher.launch_args.set_hostlist(["test", 5]) with pytest.raises(TypeError): palsLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): - palsLauncher.launch_args.set_hostlist(5) \ No newline at end of file + palsLauncher.launch_args.set_hostlist(5) diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index 758c16848c..80cfc97707 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -1,23 +1,36 @@ +import pytest + from smartsim.settings import BatchSettings -from smartsim.settings.translators.batch.pbs import QsubBatchArgBuilder from smartsim.settings.batchCommand import SchedulerType -import pytest +from smartsim.settings.translators.batch.pbs import QsubBatchArgBuilder + def test_scheduler_str(): """Ensure scheduler_str returns appropriate value""" bs = BatchSettings(batch_scheduler=SchedulerType.Pbs) assert bs.scheduler_args.scheduler_str() == SchedulerType.Pbs.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), - pytest.param("set_walltime", ("10:00:00",),"10:00:00","walltime",id="set_walltime"), - pytest.param("set_account", ("account",),"account","A",id="set_account"), - pytest.param("set_queue", ("queue",),"queue","q",id="set_queue"), - pytest.param("set_ncpus", (2,),"2","ppn",id="set_ncpus"), - pytest.param("set_hostlist", ("host_A",),"host_A","hostname",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","hostname",id="set_hostlist_list[str]"), + pytest.param("set_nodes", (2,), "2", "nodes", id="set_nodes"), + pytest.param( + "set_walltime", ("10:00:00",), "10:00:00", "walltime", id="set_walltime" + ), + pytest.param("set_account", ("account",), "account", "A", id="set_account"), + pytest.param("set_queue", ("queue",), "queue", "q", id="set_queue"), + pytest.param("set_ncpus", (2,), "2", "ppn", id="set_ncpus"), + pytest.param( + "set_hostlist", ("host_A",), "host_A", "hostname", id="set_hostlist_str" + ), + pytest.param( + "set_hostlist", + (["host_A", "host_B"],), + "host_A,host_B", + "hostname", + id="set_hostlist_list[str]", + ), ], ) def test_create_pbs_batch(function, value, flag, result): @@ -25,7 +38,8 @@ def test_create_pbs_batch(function, value, flag, result): assert isinstance(pbsScheduler.scheduler_args, QsubBatchArgBuilder) getattr(pbsScheduler.scheduler_args, function)(*value) assert pbsScheduler.scheduler_args._scheduler_args[flag] == result - + + def test_format_pbs_batch_args(): pbsScheduler = BatchSettings(batch_scheduler=SchedulerType.Pbs) pbsScheduler.scheduler_args.set_nodes(1) @@ -33,11 +47,15 @@ def test_format_pbs_batch_args(): pbsScheduler.scheduler_args.set_queue("default") pbsScheduler.scheduler_args.set_account("myproject") pbsScheduler.scheduler_args.set_ncpus(10) - pbsScheduler.scheduler_args.set_hostlist(['host_a', 'host_b', 'host_c']) + pbsScheduler.scheduler_args.set_hostlist(["host_a", "host_b", "host_c"]) args = pbsScheduler.format_batch_args() assert args == [ - "-l", "nodes=1:ncpus=10:host=host_a+host=host_b+host=host_c", - "-l", "walltime=10:00:00", - "-q", "default", - "-A", "myproject", - ] \ No newline at end of file + "-l", + "nodes=1:ncpus=10:host=host_a+host=host_b+host=host_c", + "-l", + "walltime=10:00:00", + "-q", + "default", + "-A", + "myproject", + ] diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 1fa18c00e4..f37f61e655 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,56 +1,110 @@ +import pytest + from smartsim.settings import LaunchSettings -from smartsim.settings.translators.launch.slurm import SlurmArgBuilder from smartsim.settings.launchCommand import LauncherType -import pytest +from smartsim.settings.translators.launch.slurm import SlurmArgBuilder + def test_launcher_str(): """Ensure launcher_str returns appropriate value""" ls = LaunchSettings(launcher=LauncherType.Slurm) assert ls.launch_args.launcher_str() == LauncherType.Slurm.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), - pytest.param("set_hostlist", ("host_A",),"host_A","nodelist",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","nodelist",id="set_hostlist_list[str]"), - pytest.param("set_hostlist_from_file", ("./path/to/hostfile",),"./path/to/hostfile","nodefile",id="set_hostlist_from_file"), - pytest.param("set_excluded_hosts", ("host_A",),"host_A","exclude",id="set_excluded_hosts_str"), - pytest.param("set_excluded_hosts", (["host_A","host_B"],),"host_A,host_B","exclude",id="set_excluded_hosts_list[str]"), - pytest.param("set_cpus_per_task", (4,),"4","cpus-per-task",id="set_cpus_per_task"), - pytest.param("set_tasks", (4,),"4","ntasks",id="set_tasks"), - pytest.param("set_tasks_per_node", (4,),"4","ntasks-per-node",id="set_tasks_per_node"), - pytest.param("set_cpu_bindings", (4,),"map_cpu:4","cpu_bind",id="set_cpu_bindings"), - pytest.param("set_cpu_bindings", ([4,4],),"map_cpu:4,4","cpu_bind",id="set_cpu_bindings_list[str]"), - pytest.param("set_memory_per_node", (8000,),"8000M","mem",id="set_memory_per_node"), - pytest.param("set_executable_broadcast", ("/tmp/some/path",),"/tmp/some/path","bcast",id="set_broadcast"), - pytest.param("set_node_feature", ("P100",),"P100","C",id="set_node_feature"), - pytest.param("set_walltime", ("10:00:00",),"10:00:00","time",id="set_walltime"), + pytest.param("set_nodes", (2,), "2", "nodes", id="set_nodes"), + pytest.param( + "set_hostlist", ("host_A",), "host_A", "nodelist", id="set_hostlist_str" + ), + pytest.param( + "set_hostlist", + (["host_A", "host_B"],), + "host_A,host_B", + "nodelist", + id="set_hostlist_list[str]", + ), + pytest.param( + "set_hostlist_from_file", + ("./path/to/hostfile",), + "./path/to/hostfile", + "nodefile", + id="set_hostlist_from_file", + ), + pytest.param( + "set_excluded_hosts", + ("host_A",), + "host_A", + "exclude", + id="set_excluded_hosts_str", + ), + pytest.param( + "set_excluded_hosts", + (["host_A", "host_B"],), + "host_A,host_B", + "exclude", + id="set_excluded_hosts_list[str]", + ), + pytest.param( + "set_cpus_per_task", (4,), "4", "cpus-per-task", id="set_cpus_per_task" + ), + pytest.param("set_tasks", (4,), "4", "ntasks", id="set_tasks"), + pytest.param( + "set_tasks_per_node", (4,), "4", "ntasks-per-node", id="set_tasks_per_node" + ), + pytest.param( + "set_cpu_bindings", (4,), "map_cpu:4", "cpu_bind", id="set_cpu_bindings" + ), + pytest.param( + "set_cpu_bindings", + ([4, 4],), + "map_cpu:4,4", + "cpu_bind", + id="set_cpu_bindings_list[str]", + ), + pytest.param( + "set_memory_per_node", (8000,), "8000M", "mem", id="set_memory_per_node" + ), + pytest.param( + "set_executable_broadcast", + ("/tmp/some/path",), + "/tmp/some/path", + "bcast", + id="set_broadcast", + ), + pytest.param("set_node_feature", ("P100",), "P100", "C", id="set_node_feature"), + pytest.param( + "set_walltime", ("10:00:00",), "10:00:00", "time", id="set_walltime" + ), ], ) def test_slurm_class_methods(function, value, flag, result): slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) - assert isinstance(slurmLauncher.launch_args,SlurmArgBuilder) + assert isinstance(slurmLauncher.launch_args, SlurmArgBuilder) getattr(slurmLauncher.launch_args, function)(*value) assert slurmLauncher.launch_args._launch_args[flag] == result + def test_set_verbose_launch(): ls = LaunchSettings(launcher=LauncherType.Slurm) ls.launch_args.set_verbose_launch(True) - assert ls.launch_args._launch_args == {'verbose': None} + assert ls.launch_args._launch_args == {"verbose": None} ls.launch_args.set_verbose_launch(False) assert ls.launch_args._launch_args == {} + def test_set_quiet_launch(): ls = LaunchSettings(launcher=LauncherType.Slurm) ls.launch_args.set_quiet_launch(True) - assert ls.launch_args._launch_args == {'quiet': None} + assert ls.launch_args._launch_args == {"quiet": None} ls.launch_args.set_quiet_launch(False) assert ls.launch_args._launch_args == {} - + + def test_format_env_vars(): """Test format_env_vars runs correctly""" - env_vars={ + env_vars = { "OMP_NUM_THREADS": "20", "LOGGING": "verbose", "SSKEYIN": "name_0,name_1", @@ -61,6 +115,7 @@ def test_format_env_vars(): assert "LOGGING=verbose" in ls_format assert all("SSKEYIN" not in x for x in ls_format) + def test_catch_existing_env_var(caplog, monkeypatch): slurmSettings = LaunchSettings( launcher=LauncherType.Slurm, @@ -91,9 +146,14 @@ def test_catch_existing_env_var(caplog, monkeypatch): assert record.levelname == "WARNING" assert record.message == msg + def test_format_comma_sep_env_vars(): """Test format_comma_sep_env_vars runs correctly""" - env_vars = {"OMP_NUM_THREADS": "20", "LOGGING": "verbose", "SSKEYIN": "name_0,name_1"} + env_vars = { + "OMP_NUM_THREADS": "20", + "LOGGING": "verbose", + "SSKEYIN": "name_0,name_1", + } slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm, env_vars=env_vars) formatted, comma_separated_formatted = slurmLauncher.format_comma_sep_env_vars() assert "OMP_NUM_THREADS" in formatted @@ -102,6 +162,7 @@ def test_format_comma_sep_env_vars(): assert "name_0,name_1" not in formatted assert "SSKEYIN=name_0,name_1" in comma_separated_formatted + def test_slurmSettings_settings(): """Test format_launch_args runs correctly""" slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) @@ -113,6 +174,7 @@ def test_slurmSettings_settings(): result = ["--nodes=5", "--cpus-per-task=2", "--ntasks=100", "--ntasks-per-node=20"] assert formatted == result + def test_slurmSettings_launch_args(): """Test the possible user overrides through run_args""" launch_args = { @@ -134,36 +196,40 @@ def test_slurmSettings_launch_args(): ] assert formatted == result + def test_invalid_hostlist_format(): """Test invalid hostlist formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(TypeError): - slurmLauncher.launch_args.set_hostlist(["test",5]) + slurmLauncher.launch_args.set_hostlist(["test", 5]) with pytest.raises(TypeError): slurmLauncher.launch_args.set_hostlist([5]) with pytest.raises(TypeError): slurmLauncher.launch_args.set_hostlist(5) + def test_invalid_exclude_hostlist_format(): """Test invalid hostlist formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(TypeError): - slurmLauncher.launch_args.set_excluded_hosts(["test",5]) + slurmLauncher.launch_args.set_excluded_hosts(["test", 5]) with pytest.raises(TypeError): slurmLauncher.launch_args.set_excluded_hosts([5]) with pytest.raises(TypeError): slurmLauncher.launch_args.set_excluded_hosts(5) + def test_invalid_node_feature_format(): """Test invalid node feature formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) with pytest.raises(TypeError): - slurmLauncher.launch_args.set_node_feature(["test",5]) + slurmLauncher.launch_args.set_node_feature(["test", 5]) with pytest.raises(TypeError): slurmLauncher.launch_args.set_node_feature([5]) with pytest.raises(TypeError): slurmLauncher.launch_args.set_node_feature(5) + def test_invalid_walltime_format(): """Test invalid walltime formats""" slurmLauncher = LaunchSettings(launcher=LauncherType.Slurm) @@ -176,6 +242,7 @@ def test_invalid_walltime_format(): with pytest.raises(ValueError): slurmLauncher.launch_args.set_walltime("0s:ss:ss") + def test_set_het_groups(monkeypatch): """Test ability to set one or more het groups to run setting""" monkeypatch.setenv("SLURM_HET_SIZE", "4") @@ -185,4 +252,4 @@ def test_set_het_groups(monkeypatch): slurmLauncher.launch_args.set_het_group([3, 2]) assert slurmLauncher._arg_builder._launch_args["het-group"] == "3,2" with pytest.raises(ValueError): - slurmLauncher.launch_args.set_het_group([4]) \ No newline at end of file + slurmLauncher.launch_args.set_het_group([4]) diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index f91c01eee8..ddcf0b7caa 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -1,24 +1,49 @@ +import pytest + from smartsim.settings import BatchSettings -from smartsim.settings.translators.batch.slurm import SlurmBatchArgBuilder from smartsim.settings.batchCommand import SchedulerType -import pytest +from smartsim.settings.translators.batch.slurm import SlurmBatchArgBuilder + def test_scheduler_str(): """Ensure scheduler_str returns appropriate value""" bs = BatchSettings(batch_scheduler=SchedulerType.Slurm) assert bs.scheduler_args.scheduler_str() == SchedulerType.Slurm.value + @pytest.mark.parametrize( "function,value,result,flag", [ - pytest.param("set_nodes", (2,),"2","nodes",id="set_nodes"), - pytest.param("set_walltime", ("10:00:00",),"10:00:00","time",id="set_walltime"), - pytest.param("set_account", ("account",),"account","account",id="set_account"), - pytest.param("set_partition", ("partition",),"partition","partition",id="set_partition"), - pytest.param("set_queue", ("partition",),"partition","partition",id="set_queue"), - pytest.param("set_cpus_per_task", (2,),"2","cpus-per-task",id="set_cpus_per_task"), - pytest.param("set_hostlist", ("host_A",),"host_A","nodelist",id="set_hostlist_str"), - pytest.param("set_hostlist", (["host_A","host_B"],),"host_A,host_B","nodelist",id="set_hostlist_list[str]"), + pytest.param("set_nodes", (2,), "2", "nodes", id="set_nodes"), + pytest.param( + "set_walltime", ("10:00:00",), "10:00:00", "time", id="set_walltime" + ), + pytest.param( + "set_account", ("account",), "account", "account", id="set_account" + ), + pytest.param( + "set_partition", + ("partition",), + "partition", + "partition", + id="set_partition", + ), + pytest.param( + "set_queue", ("partition",), "partition", "partition", id="set_queue" + ), + pytest.param( + "set_cpus_per_task", (2,), "2", "cpus-per-task", id="set_cpus_per_task" + ), + pytest.param( + "set_hostlist", ("host_A",), "host_A", "nodelist", id="set_hostlist_str" + ), + pytest.param( + "set_hostlist", + (["host_A", "host_B"],), + "host_A,host_B", + "nodelist", + id="set_hostlist_list[str]", + ), ], ) def test_sbatch_class_methods(function, value, flag, result): @@ -26,15 +51,16 @@ def test_sbatch_class_methods(function, value, flag, result): getattr(slurmScheduler.scheduler_args, function)(*value) assert slurmScheduler.scheduler_args._scheduler_args[flag] == result + def test_create_sbatch(): batch_args = {"exclusive": None, "oversubscribe": None} - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm, scheduler_args=batch_args) + slurmScheduler = BatchSettings( + batch_scheduler=SchedulerType.Slurm, scheduler_args=batch_args + ) assert isinstance(slurmScheduler._arg_builder, SlurmBatchArgBuilder) args = slurmScheduler.format_batch_args() - assert args == [ - "--exclusive", - "--oversubscribe" - ] + assert args == ["--exclusive", "--oversubscribe"] + def test_launch_args_input_mutation(): # Tests that the run args passed in are not modified after initialization @@ -46,7 +72,9 @@ def test_launch_args_input_mutation(): key1: val1, key2: val2, } - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm, scheduler_args=default_scheduler_args) + slurmScheduler = BatchSettings( + batch_scheduler=SchedulerType.Slurm, scheduler_args=default_scheduler_args + ) # Confirm initial values are set assert slurmScheduler.scheduler_args._scheduler_args[key0] == val0 @@ -60,9 +88,12 @@ def test_launch_args_input_mutation(): # Confirm previously created run settings are not changed assert slurmScheduler.scheduler_args._scheduler_args[key2] == val2 + def test_sbatch_settings(): scheduler_args = {"nodes": 1, "time": "10:00:00", "account": "A3123"} - slurmScheduler = BatchSettings(batch_scheduler=SchedulerType.Slurm,scheduler_args=scheduler_args) + slurmScheduler = BatchSettings( + batch_scheduler=SchedulerType.Slurm, scheduler_args=scheduler_args + ) formatted = slurmScheduler.format_batch_args() result = ["--nodes=1", "--time=10:00:00", "--account=A3123"] assert formatted == result @@ -76,4 +107,4 @@ def test_sbatch_manual(): formatted = slurmScheduler.format_batch_args() print(f"here: {formatted}") result = ["--nodes=5", "--account=A3531", "--time=10:00:00"] - assert formatted == result \ No newline at end of file + assert formatted == result From 1ea448925cab27499c2a59461581ef0eecd45f02 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 11 Jun 2024 17:16:36 -0500 Subject: [PATCH 37/43] c --- smartsim/_core/commands/__init__.py | 2 +- smartsim/settings/baseSettings.py | 2 +- smartsim/settings/translators/batch/__init.__.py | 2 +- smartsim/settings/translators/launch/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/smartsim/_core/commands/__init__.py b/smartsim/_core/commands/__init__.py index 739f8563b3..72ef1f674a 100644 --- a/smartsim/_core/commands/__init__.py +++ b/smartsim/_core/commands/__init__.py @@ -26,4 +26,4 @@ from .command import Command from .commandList import CommandList -from .launchCommands import LaunchCommands \ No newline at end of file +from .launchCommands import LaunchCommands diff --git a/smartsim/settings/baseSettings.py b/smartsim/settings/baseSettings.py index b4c998757d..1acd5f6057 100644 --- a/smartsim/settings/baseSettings.py +++ b/smartsim/settings/baseSettings.py @@ -26,4 +26,4 @@ # fmt: off class BaseSettings: ... -# fmt: on \ No newline at end of file +# fmt: on diff --git a/smartsim/settings/translators/batch/__init.__.py b/smartsim/settings/translators/batch/__init.__.py index a232d8866b..41dcbbfc2c 100644 --- a/smartsim/settings/translators/batch/__init.__.py +++ b/smartsim/settings/translators/batch/__init.__.py @@ -32,4 +32,4 @@ "BsubBatchArgBuilder", "QsubBatchArgBuilder", "SlurmBatchArgBuilder", -] \ No newline at end of file +] diff --git a/smartsim/settings/translators/launch/__init__.py b/smartsim/settings/translators/launch/__init__.py index 7cc9844ba4..d593c59f7c 100644 --- a/smartsim/settings/translators/launch/__init__.py +++ b/smartsim/settings/translators/launch/__init__.py @@ -16,4 +16,4 @@ "OrteArgBuilder", "PalsMpiexecArgBuilder", "SlurmArgBuilder", -] \ No newline at end of file +] From b862095ec97e857523b7bd32952425727f1c1b04 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 11 Jun 2024 17:32:36 -0500 Subject: [PATCH 38/43] init changes --- smartsim/_core/utils/helpers.py | 2 +- smartsim/settings/batchSettings.py | 11 +++++------ .../{translators => builders}/__init__.py | 0 .../batch/__init.__.py | 0 .../{translators => builders}/batch/lsf.py | 5 ----- .../{translators => builders}/batch/pbs.py | 5 ----- .../{translators => builders}/batch/slurm.py | 5 ----- .../batchArgBuilder.py | 0 .../launch/__init__.py | 0 .../{translators => builders}/launch/alps.py | 7 +------ .../{translators => builders}/launch/dragon.py | 5 ----- .../{translators => builders}/launch/local.py | 5 ----- .../{translators => builders}/launch/lsf.py | 5 ----- .../{translators => builders}/launch/mpi.py | 5 ----- .../{translators => builders}/launch/pals.py | 5 ----- .../{translators => builders}/launch/slurm.py | 7 +------ .../launchArgBuilder.py | 0 smartsim/settings/launchSettings.py | 17 ++++++++--------- .../test_settings/test_alpsLauncher.py | 2 +- .../test_settings/test_dragonLauncher.py | 2 +- .../test_settings/test_launchSettings.py | 10 +--------- .../test_settings/test_localLauncher.py | 2 +- .../test_settings/test_lsfLauncher.py | 2 +- .../test_settings/test_mpiLauncher.py | 2 +- .../test_settings/test_palsLauncher.py | 2 +- .../test_settings/test_pbsScheduler.py | 2 +- .../test_settings/test_slurmLauncher.py | 2 +- .../test_settings/test_slurmScheduler.py | 2 +- 28 files changed, 26 insertions(+), 86 deletions(-) rename smartsim/settings/{translators => builders}/__init__.py (100%) rename smartsim/settings/{translators => builders}/batch/__init.__.py (100%) rename smartsim/settings/{translators => builders}/batch/lsf.py (97%) rename smartsim/settings/{translators => builders}/batch/pbs.py (97%) rename smartsim/settings/{translators => builders}/batch/slurm.py (97%) rename smartsim/settings/{translators => builders}/batchArgBuilder.py (100%) rename smartsim/settings/{translators => builders}/launch/__init__.py (100%) rename smartsim/settings/{translators => builders}/launch/alps.py (97%) rename smartsim/settings/{translators => builders}/launch/dragon.py (94%) rename smartsim/settings/{translators => builders}/launch/local.py (95%) rename smartsim/settings/{translators => builders}/launch/lsf.py (96%) rename smartsim/settings/{translators => builders}/launch/mpi.py (98%) rename smartsim/settings/{translators => builders}/launch/pals.py (97%) rename smartsim/settings/{translators => builders}/launch/slurm.py (98%) rename smartsim/settings/{translators => builders}/launchArgBuilder.py (100%) diff --git a/smartsim/_core/utils/helpers.py b/smartsim/_core/utils/helpers.py index 947e3ee644..70f52bc4e1 100644 --- a/smartsim/_core/utils/helpers.py +++ b/smartsim/_core/utils/helpers.py @@ -87,7 +87,7 @@ def check_dev_log_level() -> bool: return lvl == "developer" -def fmt_dict(value: t.Dict[str, t.Any] | t.Mapping[str, t.Any]) -> str: +def fmt_dict(value: t.Mapping[str, t.Any]) -> str: fmt_str = "" for k, v in value.items(): fmt_str += "\t" + str(k) + " = " + str(v) diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index 5478069120..fa073f2a7c 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -35,10 +35,10 @@ from .baseSettings import BaseSettings from .batchCommand import SchedulerType from .common import StringArgument -from .translators import BatchArgBuilder -from .translators.batch.lsf import BsubBatchArgBuilder -from .translators.batch.pbs import QsubBatchArgBuilder -from .translators.batch.slurm import SlurmBatchArgBuilder +from .builders import BatchArgBuilder +from .builders.batch.lsf import BsubBatchArgBuilder +from .builders.batch.pbs import QsubBatchArgBuilder +from .builders.batch.slurm import SlurmBatchArgBuilder logger = get_logger(__name__) @@ -53,7 +53,7 @@ def __init__( try: self._batch_scheduler = SchedulerType(batch_scheduler) except ValueError: - raise ValueError(f"Invalid scheduler type: {batch_scheduler}") + raise ValueError(f"Invalid scheduler type: {batch_scheduler}") from None self._arg_builder = self._get_arg_builder(scheduler_args) self.env_vars = env_vars or {} @@ -70,7 +70,6 @@ def batch_scheduler(self) -> str: @property def scheduler_args(self) -> BatchArgBuilder: """Return the batch argument translator.""" - # Is a deep copy needed here? return self._arg_builder @property diff --git a/smartsim/settings/translators/__init__.py b/smartsim/settings/builders/__init__.py similarity index 100% rename from smartsim/settings/translators/__init__.py rename to smartsim/settings/builders/__init__.py diff --git a/smartsim/settings/translators/batch/__init.__.py b/smartsim/settings/builders/batch/__init.__.py similarity index 100% rename from smartsim/settings/translators/batch/__init.__.py rename to smartsim/settings/builders/batch/__init.__.py diff --git a/smartsim/settings/translators/batch/lsf.py b/smartsim/settings/builders/batch/lsf.py similarity index 97% rename from smartsim/settings/translators/batch/lsf.py rename to smartsim/settings/builders/batch/lsf.py index 9d44b2c2ad..8f151885a9 100644 --- a/smartsim/settings/translators/batch/lsf.py +++ b/smartsim/settings/builders/batch/lsf.py @@ -38,11 +38,6 @@ class BsubBatchArgBuilder(BatchArgBuilder): - def __init__( - self, - scheduler_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(scheduler_args) def scheduler_str(self) -> str: """Get the string representation of the scheduler""" diff --git a/smartsim/settings/translators/batch/pbs.py b/smartsim/settings/builders/batch/pbs.py similarity index 97% rename from smartsim/settings/translators/batch/pbs.py rename to smartsim/settings/builders/batch/pbs.py index f04c338106..39582f99e8 100644 --- a/smartsim/settings/translators/batch/pbs.py +++ b/smartsim/settings/builders/batch/pbs.py @@ -40,11 +40,6 @@ class QsubBatchArgBuilder(BatchArgBuilder): - def __init__( - self, - scheduler_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(scheduler_args) def scheduler_str(self) -> str: """Get the string representation of the scheduler""" diff --git a/smartsim/settings/translators/batch/slurm.py b/smartsim/settings/builders/batch/slurm.py similarity index 97% rename from smartsim/settings/translators/batch/slurm.py rename to smartsim/settings/builders/batch/slurm.py index 4a24ce6150..c15e185f66 100644 --- a/smartsim/settings/translators/batch/slurm.py +++ b/smartsim/settings/builders/batch/slurm.py @@ -39,11 +39,6 @@ class SlurmBatchArgBuilder(BatchArgBuilder): - def __init__( - self, - scheduler_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(scheduler_args) def scheduler_str(self) -> str: """Get the string representation of the scheduler""" diff --git a/smartsim/settings/translators/batchArgBuilder.py b/smartsim/settings/builders/batchArgBuilder.py similarity index 100% rename from smartsim/settings/translators/batchArgBuilder.py rename to smartsim/settings/builders/batchArgBuilder.py diff --git a/smartsim/settings/translators/launch/__init__.py b/smartsim/settings/builders/launch/__init__.py similarity index 100% rename from smartsim/settings/translators/launch/__init__.py rename to smartsim/settings/builders/launch/__init__.py diff --git a/smartsim/settings/translators/launch/alps.py b/smartsim/settings/builders/launch/alps.py similarity index 97% rename from smartsim/settings/translators/launch/alps.py rename to smartsim/settings/builders/launch/alps.py index 1c562344a5..08b06ab732 100644 --- a/smartsim/settings/translators/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -38,11 +38,6 @@ class AprunArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" @@ -194,7 +189,7 @@ def format_launch_args(self) -> t.Union[t.List[str], None]: # args launcher uses args = [] for opt, value in self._launch_args.items(): - short_arg = bool(len(str(opt)) == 1) + short_arg = len(opt) == 1 prefix = "-" if short_arg else "--" if not value: args += [prefix + opt] diff --git a/smartsim/settings/translators/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py similarity index 94% rename from smartsim/settings/translators/launch/dragon.py rename to smartsim/settings/builders/launch/dragon.py index 76c6e60a76..90bc14060d 100644 --- a/smartsim/settings/translators/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -38,11 +38,6 @@ class DragonArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/translators/launch/local.py b/smartsim/settings/builders/launch/local.py similarity index 95% rename from smartsim/settings/translators/launch/local.py rename to smartsim/settings/builders/launch/local.py index f664fb127a..c21e941d13 100644 --- a/smartsim/settings/translators/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -38,11 +38,6 @@ class LocalArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/translators/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py similarity index 96% rename from smartsim/settings/translators/launch/lsf.py rename to smartsim/settings/builders/launch/lsf.py index 4d186e780e..9c98d92eab 100644 --- a/smartsim/settings/translators/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -38,11 +38,6 @@ class JsrunArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/translators/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py similarity index 98% rename from smartsim/settings/translators/launch/mpi.py rename to smartsim/settings/builders/launch/mpi.py index e5c7194235..2717df53f4 100644 --- a/smartsim/settings/translators/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -38,11 +38,6 @@ class _BaseMPIArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" diff --git a/smartsim/settings/translators/launch/pals.py b/smartsim/settings/builders/launch/pals.py similarity index 97% rename from smartsim/settings/translators/launch/pals.py rename to smartsim/settings/builders/launch/pals.py index 98481e8c7e..fc48e330c2 100644 --- a/smartsim/settings/translators/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -38,11 +38,6 @@ class PalsMpiexecArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/translators/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py similarity index 98% rename from smartsim/settings/translators/launch/slurm.py rename to smartsim/settings/builders/launch/slurm.py index 6f6ad40692..1e66e5db5c 100644 --- a/smartsim/settings/translators/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -32,7 +32,7 @@ from smartsim.log import get_logger -from ...common import IntegerArgument, StringArgument, set_check_input +from ...common import set_check_input from ...launchCommand import LauncherType from ..launchArgBuilder import LaunchArgBuilder @@ -40,11 +40,6 @@ class SlurmArgBuilder(LaunchArgBuilder): - def __init__( - self, - launch_args: t.Dict[str, str | None] | None, - ) -> None: - super().__init__(launch_args) def launcher_str(self) -> str: """Get the string representation of the launcher""" diff --git a/smartsim/settings/translators/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py similarity index 100% rename from smartsim/settings/translators/launchArgBuilder.py rename to smartsim/settings/builders/launchArgBuilder.py diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 9cc133766c..57ee219b2c 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -35,14 +35,14 @@ from .baseSettings import BaseSettings from .common import StringArgument from .launchCommand import LauncherType -from .translators import LaunchArgBuilder -from .translators.launch.alps import AprunArgBuilder -from .translators.launch.dragon import DragonArgBuilder -from .translators.launch.local import LocalArgBuilder -from .translators.launch.lsf import JsrunArgBuilder -from .translators.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder -from .translators.launch.pals import PalsMpiexecArgBuilder -from .translators.launch.slurm import SlurmArgBuilder +from .builders import LaunchArgBuilder +from .builders.launch.alps import AprunArgBuilder +from .builders.launch.dragon import DragonArgBuilder +from .builders.launch.local import LocalArgBuilder +from .builders.launch.lsf import JsrunArgBuilder +from .builders.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder +from .builders.launch.pals import PalsMpiexecArgBuilder +from .builders.launch.slurm import SlurmArgBuilder logger = get_logger(__name__) @@ -69,7 +69,6 @@ def launcher(self) -> str: @property def launch_args(self) -> LaunchArgBuilder: """Return the launch argument translator.""" - # Is a deep copy needed here? return self._arg_builder @launch_args.setter diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index 2fcb85e2a5..c004ec3c09 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -2,7 +2,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.alps import AprunArgBuilder +from smartsim.settings.builders.launch.alps import AprunArgBuilder def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 994e90f5cc..9de6265b64 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -2,7 +2,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.dragon import DragonArgBuilder +from smartsim.settings.builders.launch.dragon import DragonArgBuilder def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py index ff5e2dd67f..58154ffca6 100644 --- a/tests/temp_tests/test_settings/test_launchSettings.py +++ b/tests/temp_tests/test_settings/test_launchSettings.py @@ -9,15 +9,7 @@ @pytest.mark.parametrize( "launch_enum", [ - pytest.param(LauncherType.Slurm, id="slurm"), - pytest.param(LauncherType.Dragon, id="dragon"), - pytest.param(LauncherType.Pals, id="pals"), - pytest.param(LauncherType.Alps, id="alps"), - pytest.param(LauncherType.Local, id="local"), - pytest.param(LauncherType.Mpiexec, id="mpiexec"), - pytest.param(LauncherType.Mpirun, id="mpirun"), - pytest.param(LauncherType.Orterun, id="orterun"), - pytest.param(LauncherType.Lsf, id="lsf"), + pytest.param(type_,id=type_.value) for type_ in LauncherType ], ) def test_create_launch_settings(launch_enum): diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index b0ec3328b4..68e2bca772 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -2,7 +2,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.local import LocalArgBuilder +from smartsim.settings.builders.launch.local import LocalArgBuilder def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index f87b177ece..fbaecacf61 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -2,7 +2,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.lsf import JsrunArgBuilder +from smartsim.settings.builders.launch.lsf import JsrunArgBuilder def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index 9de34cca50..f5c6eff769 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -4,7 +4,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.mpi import ( +from smartsim.settings.builders.launch.mpi import ( MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder, diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 5fbbccab77..0b88db9ca5 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -2,7 +2,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.pals import PalsMpiexecArgBuilder +from smartsim.settings.builders.launch.pals import PalsMpiexecArgBuilder def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_pbsScheduler.py b/tests/temp_tests/test_settings/test_pbsScheduler.py index 80cfc97707..ab3435df55 100644 --- a/tests/temp_tests/test_settings/test_pbsScheduler.py +++ b/tests/temp_tests/test_settings/test_pbsScheduler.py @@ -2,7 +2,7 @@ from smartsim.settings import BatchSettings from smartsim.settings.batchCommand import SchedulerType -from smartsim.settings.translators.batch.pbs import QsubBatchArgBuilder +from smartsim.settings.builders.batch.pbs import QsubBatchArgBuilder def test_scheduler_str(): diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index f37f61e655..0db9a2fc02 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -2,7 +2,7 @@ from smartsim.settings import LaunchSettings from smartsim.settings.launchCommand import LauncherType -from smartsim.settings.translators.launch.slurm import SlurmArgBuilder +from smartsim.settings.builders.launch.slurm import SlurmArgBuilder def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_slurmScheduler.py b/tests/temp_tests/test_settings/test_slurmScheduler.py index ddcf0b7caa..0a34b6473f 100644 --- a/tests/temp_tests/test_settings/test_slurmScheduler.py +++ b/tests/temp_tests/test_settings/test_slurmScheduler.py @@ -2,7 +2,7 @@ from smartsim.settings import BatchSettings from smartsim.settings.batchCommand import SchedulerType -from smartsim.settings.translators.batch.slurm import SlurmBatchArgBuilder +from smartsim.settings.builders.batch.slurm import SlurmBatchArgBuilder def test_scheduler_str(): From 0e9c287bd52bf0c79ac6ac936a838df2810dc3f7 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 11 Jun 2024 17:38:28 -0500 Subject: [PATCH 39/43] isort will be the death of me --- smartsim/settings/batchSettings.py | 2 +- smartsim/settings/builders/batch/lsf.py | 1 - smartsim/settings/builders/batch/pbs.py | 1 - smartsim/settings/builders/batch/slurm.py | 1 - smartsim/settings/builders/launch/alps.py | 1 - smartsim/settings/builders/launch/dragon.py | 1 - smartsim/settings/builders/launch/local.py | 1 - smartsim/settings/builders/launch/lsf.py | 1 - smartsim/settings/builders/launch/mpi.py | 1 - smartsim/settings/builders/launch/pals.py | 1 - smartsim/settings/builders/launch/slurm.py | 1 - smartsim/settings/launchSettings.py | 4 ++-- tests/temp_tests/test_settings/test_alpsLauncher.py | 2 +- tests/temp_tests/test_settings/test_dragonLauncher.py | 2 +- tests/temp_tests/test_settings/test_launchSettings.py | 4 +--- tests/temp_tests/test_settings/test_localLauncher.py | 2 +- tests/temp_tests/test_settings/test_lsfLauncher.py | 2 +- tests/temp_tests/test_settings/test_mpiLauncher.py | 2 +- tests/temp_tests/test_settings/test_palsLauncher.py | 2 +- tests/temp_tests/test_settings/test_slurmLauncher.py | 2 +- 20 files changed, 11 insertions(+), 23 deletions(-) diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index fa073f2a7c..ebf0499bf5 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -34,11 +34,11 @@ from .._core.utils.helpers import fmt_dict from .baseSettings import BaseSettings from .batchCommand import SchedulerType -from .common import StringArgument from .builders import BatchArgBuilder from .builders.batch.lsf import BsubBatchArgBuilder from .builders.batch.pbs import QsubBatchArgBuilder from .builders.batch.slurm import SlurmBatchArgBuilder +from .common import StringArgument logger = get_logger(__name__) diff --git a/smartsim/settings/builders/batch/lsf.py b/smartsim/settings/builders/batch/lsf.py index 8f151885a9..4bb7bbd27a 100644 --- a/smartsim/settings/builders/batch/lsf.py +++ b/smartsim/settings/builders/batch/lsf.py @@ -38,7 +38,6 @@ class BsubBatchArgBuilder(BatchArgBuilder): - def scheduler_str(self) -> str: """Get the string representation of the scheduler""" return SchedulerType.Lsf.value diff --git a/smartsim/settings/builders/batch/pbs.py b/smartsim/settings/builders/batch/pbs.py index 39582f99e8..d04b4beba1 100644 --- a/smartsim/settings/builders/batch/pbs.py +++ b/smartsim/settings/builders/batch/pbs.py @@ -40,7 +40,6 @@ class QsubBatchArgBuilder(BatchArgBuilder): - def scheduler_str(self) -> str: """Get the string representation of the scheduler""" return SchedulerType.Pbs.value diff --git a/smartsim/settings/builders/batch/slurm.py b/smartsim/settings/builders/batch/slurm.py index c15e185f66..5a03f5acd1 100644 --- a/smartsim/settings/builders/batch/slurm.py +++ b/smartsim/settings/builders/batch/slurm.py @@ -39,7 +39,6 @@ class SlurmBatchArgBuilder(BatchArgBuilder): - def scheduler_str(self) -> str: """Get the string representation of the scheduler""" return SchedulerType.Slurm.value diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index 08b06ab732..99ab3967fa 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -38,7 +38,6 @@ class AprunArgBuilder(LaunchArgBuilder): - def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wdir"} diff --git a/smartsim/settings/builders/launch/dragon.py b/smartsim/settings/builders/launch/dragon.py index 90bc14060d..1ca0a244de 100644 --- a/smartsim/settings/builders/launch/dragon.py +++ b/smartsim/settings/builders/launch/dragon.py @@ -38,7 +38,6 @@ class DragonArgBuilder(LaunchArgBuilder): - def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Dragon.value diff --git a/smartsim/settings/builders/launch/local.py b/smartsim/settings/builders/launch/local.py index c21e941d13..595514f155 100644 --- a/smartsim/settings/builders/launch/local.py +++ b/smartsim/settings/builders/launch/local.py @@ -38,7 +38,6 @@ class LocalArgBuilder(LaunchArgBuilder): - def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Local.value diff --git a/smartsim/settings/builders/launch/lsf.py b/smartsim/settings/builders/launch/lsf.py index 9c98d92eab..2c72002e54 100644 --- a/smartsim/settings/builders/launch/lsf.py +++ b/smartsim/settings/builders/launch/lsf.py @@ -38,7 +38,6 @@ class JsrunArgBuilder(LaunchArgBuilder): - def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Lsf.value diff --git a/smartsim/settings/builders/launch/mpi.py b/smartsim/settings/builders/launch/mpi.py index 2717df53f4..1331be317f 100644 --- a/smartsim/settings/builders/launch/mpi.py +++ b/smartsim/settings/builders/launch/mpi.py @@ -38,7 +38,6 @@ class _BaseMPIArgBuilder(LaunchArgBuilder): - def _reserved_launch_args(self) -> set[str]: """Return reserved launch arguments.""" return {"wd", "wdir"} diff --git a/smartsim/settings/builders/launch/pals.py b/smartsim/settings/builders/launch/pals.py index fc48e330c2..051409c295 100644 --- a/smartsim/settings/builders/launch/pals.py +++ b/smartsim/settings/builders/launch/pals.py @@ -38,7 +38,6 @@ class PalsMpiexecArgBuilder(LaunchArgBuilder): - def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Pals.value diff --git a/smartsim/settings/builders/launch/slurm.py b/smartsim/settings/builders/launch/slurm.py index 1e66e5db5c..80d3d6be28 100644 --- a/smartsim/settings/builders/launch/slurm.py +++ b/smartsim/settings/builders/launch/slurm.py @@ -40,7 +40,6 @@ class SlurmArgBuilder(LaunchArgBuilder): - def launcher_str(self) -> str: """Get the string representation of the launcher""" return LauncherType.Slurm.value diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index 57ee219b2c..f37c71c199 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -33,8 +33,6 @@ from .._core.utils.helpers import fmt_dict from .baseSettings import BaseSettings -from .common import StringArgument -from .launchCommand import LauncherType from .builders import LaunchArgBuilder from .builders.launch.alps import AprunArgBuilder from .builders.launch.dragon import DragonArgBuilder @@ -43,6 +41,8 @@ from .builders.launch.mpi import MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder from .builders.launch.pals import PalsMpiexecArgBuilder from .builders.launch.slurm import SlurmArgBuilder +from .common import StringArgument +from .launchCommand import LauncherType logger = get_logger(__name__) diff --git a/tests/temp_tests/test_settings/test_alpsLauncher.py b/tests/temp_tests/test_settings/test_alpsLauncher.py index c004ec3c09..7f9a4c3b96 100644 --- a/tests/temp_tests/test_settings/test_alpsLauncher.py +++ b/tests/temp_tests/test_settings/test_alpsLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.alps import AprunArgBuilder +from smartsim.settings.launchCommand import LauncherType def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_dragonLauncher.py b/tests/temp_tests/test_settings/test_dragonLauncher.py index 9de6265b64..d21a21c598 100644 --- a/tests/temp_tests/test_settings/test_dragonLauncher.py +++ b/tests/temp_tests/test_settings/test_dragonLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.dragon import DragonArgBuilder +from smartsim.settings.launchCommand import LauncherType def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_launchSettings.py b/tests/temp_tests/test_settings/test_launchSettings.py index 58154ffca6..b84f013d78 100644 --- a/tests/temp_tests/test_settings/test_launchSettings.py +++ b/tests/temp_tests/test_settings/test_launchSettings.py @@ -8,9 +8,7 @@ @pytest.mark.parametrize( "launch_enum", - [ - pytest.param(type_,id=type_.value) for type_ in LauncherType - ], + [pytest.param(type_, id=type_.value) for type_ in LauncherType], ) def test_create_launch_settings(launch_enum): ls_str = LaunchSettings( diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index 68e2bca772..ae419cdb66 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.local import LocalArgBuilder +from smartsim.settings.launchCommand import LauncherType def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_lsfLauncher.py b/tests/temp_tests/test_settings/test_lsfLauncher.py index fbaecacf61..4c4260ac50 100644 --- a/tests/temp_tests/test_settings/test_lsfLauncher.py +++ b/tests/temp_tests/test_settings/test_lsfLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.lsf import JsrunArgBuilder +from smartsim.settings.launchCommand import LauncherType def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_mpiLauncher.py b/tests/temp_tests/test_settings/test_mpiLauncher.py index f5c6eff769..815f0c5c13 100644 --- a/tests/temp_tests/test_settings/test_mpiLauncher.py +++ b/tests/temp_tests/test_settings/test_mpiLauncher.py @@ -3,12 +3,12 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.mpi import ( MpiArgBuilder, MpiexecArgBuilder, OrteArgBuilder, ) +from smartsim.settings.launchCommand import LauncherType @pytest.mark.parametrize( diff --git a/tests/temp_tests/test_settings/test_palsLauncher.py b/tests/temp_tests/test_settings/test_palsLauncher.py index 0b88db9ca5..01cbea2ed6 100644 --- a/tests/temp_tests/test_settings/test_palsLauncher.py +++ b/tests/temp_tests/test_settings/test_palsLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.pals import PalsMpiexecArgBuilder +from smartsim.settings.launchCommand import LauncherType def test_launcher_str(): diff --git a/tests/temp_tests/test_settings/test_slurmLauncher.py b/tests/temp_tests/test_settings/test_slurmLauncher.py index 0db9a2fc02..c5e9b5b62d 100644 --- a/tests/temp_tests/test_settings/test_slurmLauncher.py +++ b/tests/temp_tests/test_settings/test_slurmLauncher.py @@ -1,8 +1,8 @@ import pytest from smartsim.settings import LaunchSettings -from smartsim.settings.launchCommand import LauncherType from smartsim.settings.builders.launch.slurm import SlurmArgBuilder +from smartsim.settings.launchCommand import LauncherType def test_launcher_str(): From 2765805c5f5516c272d75e7dd0700956a4dd1ef1 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Tue, 11 Jun 2024 19:12:46 -0500 Subject: [PATCH 40/43] commit --- smartsim/_core/commands/launchCommands.py | 6 +++--- smartsim/settings/batchSettings.py | 2 +- smartsim/settings/builders/batchArgBuilder.py | 5 +++++ smartsim/settings/builders/launch/alps.py | 2 +- smartsim/settings/builders/launchArgBuilder.py | 5 +++++ smartsim/settings/common.py | 7 ++++--- smartsim/settings/launchCommand.py | 1 - smartsim/settings/launchSettings.py | 10 ++++++---- tests/temp_tests/test_settings/test_batchSettings.py | 1 + tests/temp_tests/test_settings/test_common.py | 10 ++++++++++ tests/temp_tests/test_settings/test_localLauncher.py | 1 - 11 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 tests/temp_tests/test_settings/test_common.py diff --git a/smartsim/_core/commands/launchCommands.py b/smartsim/_core/commands/launchCommands.py index 53d3da6f84..c62186671b 100644 --- a/smartsim/_core/commands/launchCommands.py +++ b/smartsim/_core/commands/launchCommands.py @@ -40,12 +40,12 @@ def postlaunch_command(self) -> CommandList: def __str__(self) -> str: # pragma: no cover string = "\n\nPrelaunch Command List:\n" - for _, pre_cmd in enumerate(self.prelaunch_command): + for pre_cmd in self.prelaunch_command: string += f"{pre_cmd}\n" string += "\n\nLaunch Command List:\n" - for _, launch_cmd in enumerate(self.launch_command): + for launch_cmd in self.launch_command: string += f"{launch_cmd}\n" string += "\n\nPostlaunch Command List:\n" - for _, post_cmd in enumerate(self.postlaunch_command): + for post_cmd in self.postlaunch_command: string += f"{post_cmd}\n" return string diff --git a/smartsim/settings/batchSettings.py b/smartsim/settings/batchSettings.py index ebf0499bf5..efa7c4b477 100644 --- a/smartsim/settings/batchSettings.py +++ b/smartsim/settings/batchSettings.py @@ -105,7 +105,7 @@ def format_batch_args(self) -> t.List[str]: def __str__(self) -> str: # pragma: no-cover string = f"\nScheduler: {self.scheduler}" if self.scheduler_args: - string += f"\nScheduler Arguments:\n{fmt_dict(self.scheduler_args._scheduler_args)}" + string += str(self.scheduler_args) if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" return string diff --git a/smartsim/settings/builders/batchArgBuilder.py b/smartsim/settings/builders/batchArgBuilder.py index 50426559d4..117bf49e5c 100644 --- a/smartsim/settings/builders/batchArgBuilder.py +++ b/smartsim/settings/builders/batchArgBuilder.py @@ -28,6 +28,7 @@ import copy import typing as t +from ..._core.utils.helpers import fmt_dict from abc import ABC, abstractmethod from smartsim.log import get_logger @@ -100,3 +101,7 @@ def format_batch_args(self) -> t.List[str]: :return: batch arguments for Sbatch """ pass + + def __str__(self) -> str: # pragma: no-cover + string = f"\nScheduler Arguments:\n{fmt_dict(self._scheduler_args)}" + return string \ No newline at end of file diff --git a/smartsim/settings/builders/launch/alps.py b/smartsim/settings/builders/launch/alps.py index 99ab3967fa..a527cafac0 100644 --- a/smartsim/settings/builders/launch/alps.py +++ b/smartsim/settings/builders/launch/alps.py @@ -163,7 +163,7 @@ def set_quiet_launch(self, quiet: bool) -> None: :param quiet: Whether the job should be run quietly """ if quiet: - self.set("quiet", None) + self._launch_args["quiet"] = None else: self._launch_args.pop("quiet", None) diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index f3560e140e..3ada3ad7b5 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -31,6 +31,7 @@ from abc import ABC, abstractmethod from smartsim.log import get_logger +from ..._core.utils.helpers import fmt_dict logger = get_logger(__name__) @@ -87,3 +88,7 @@ def format_env_vars( """ logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") return None + + def __str__(self) -> str: # pragma: no-cover + string = f"\nLaunch Arguments:\n{fmt_dict(self._launch_args)}" + return string \ No newline at end of file diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index 66cf4f7386..2670076678 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -35,9 +35,10 @@ def set_check_input(key: str, value: str | None) -> None: - # TODO check this - if not (isinstance(key, str) or isinstance(value, (str, None))): - raise TypeError("Argument name should be of type str") + if not isinstance(key, str): + raise TypeError(f"Key '{key}' should be of type str") + if not isinstance(value, (str, type(None))): + raise TypeError(f"Value '{value}' should be of type str or None") if key.startswith("-"): key = key.lstrip("-") logger.warning( diff --git a/smartsim/settings/launchCommand.py b/smartsim/settings/launchCommand.py index 491f01d867..6f4e1aae8a 100644 --- a/smartsim/settings/launchCommand.py +++ b/smartsim/settings/launchCommand.py @@ -26,7 +26,6 @@ from enum import Enum - class LauncherType(Enum): """Launchers that are supported by SmartSim. diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index f37c71c199..c29240228b 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -125,8 +125,10 @@ def update_env(self, env_vars: t.Dict[str, str | None]) -> None: """ # Coerce env_vars values to str as a convenience to user for env, val in env_vars.items(): - if not (isinstance(val, str) and isinstance(env, str)): - raise TypeError(f"env_vars[{env}] was of type {type(val)}, not str") + if not isinstance(env, str): + raise TypeError(f"The key '{env}' of env_vars should be of type str") + if not isinstance(val, (str, type(None))): + raise TypeError(f"The value '{val}' of env_vars should be of type str or None") self._env_vars.update(env_vars) def format_env_vars(self) -> t.Union[t.List[str], None]: @@ -154,8 +156,8 @@ def format_launch_args(self) -> t.Union[t.List[str], None]: def __str__(self) -> str: # pragma: no-cover string = f"\nLauncher: {self.launcher}" - if self.launch_args._launch_args: - string += f"\nLaunch Arguments:\n{fmt_dict(self.launch_args._launch_args)}" + if self.launch_args: + string += str(self.launch_args) if self.env_vars: string += f"\nEnvironment variables: \n{fmt_dict(self.env_vars)}" return string diff --git a/tests/temp_tests/test_settings/test_batchSettings.py b/tests/temp_tests/test_settings/test_batchSettings.py index ab3490a765..81c74c1aa9 100644 --- a/tests/temp_tests/test_settings/test_batchSettings.py +++ b/tests/temp_tests/test_settings/test_batchSettings.py @@ -18,6 +18,7 @@ def test_create_scheduler_settings(scheduler_enum): scheduler_args={"launch": "var"}, env_vars={"ENV": "VAR"}, ) + print(bs_str) assert bs_str._batch_scheduler == scheduler_enum # TODO need to test scheduler_args assert bs_str._env_vars == {"ENV": "VAR"} diff --git a/tests/temp_tests/test_settings/test_common.py b/tests/temp_tests/test_settings/test_common.py new file mode 100644 index 0000000000..d163275904 --- /dev/null +++ b/tests/temp_tests/test_settings/test_common.py @@ -0,0 +1,10 @@ +from smartsim.settings.common import set_check_input +import pytest + +def test_check_set_raise_error(): + with pytest.raises(TypeError): + set_check_input(key="test", value=3) + with pytest.raises(TypeError): + set_check_input(key=3, value="str") + with pytest.raises(TypeError): + set_check_input(key=2, value=None) \ No newline at end of file diff --git a/tests/temp_tests/test_settings/test_localLauncher.py b/tests/temp_tests/test_settings/test_localLauncher.py index ae419cdb66..1ee7b9d87b 100644 --- a/tests/temp_tests/test_settings/test_localLauncher.py +++ b/tests/temp_tests/test_settings/test_localLauncher.py @@ -65,7 +65,6 @@ def test_format_launch_args(): @pytest.mark.parametrize( "env_vars", [ - pytest.param({"env1": None}, id="null value not allowed"), pytest.param({"env1": {"abc"}}, id="set value not allowed"), pytest.param({"env1": {"abc": "def"}}, id="dict value not allowed"), ], From bf87114bdd66e3910c459b7eccc49c9486e3b7c0 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 12 Jun 2024 12:45:59 -0500 Subject: [PATCH 41/43] chnages --- smartsim/entity/model.py | 3 +- smartsim/launchable/job.py | 12 +-- smartsim/launchable/mpmdjob.py | 8 +- smartsim/launchable/mpmdpair.py | 6 +- smartsim/settings/builders/batchArgBuilder.py | 5 +- .../settings/builders/launchArgBuilder.py | 6 +- smartsim/settings/launchCommand.py | 1 + smartsim/settings/launchSettings.py | 4 +- smartsim/wlm/slurm.py | 2 +- tests/temp_tests/test_colocatedJobGroup.py | 18 ++-- tests/temp_tests/test_jobGroup.py | 18 ++-- tests/temp_tests/test_launchable.py | 102 +++++++++++------- tests/temp_tests/test_settings/test_common.py | 6 +- 13 files changed, 107 insertions(+), 84 deletions(-) diff --git a/smartsim/entity/model.py b/smartsim/entity/model.py index 7587b19b0b..95673d2548 100644 --- a/smartsim/entity/model.py +++ b/smartsim/entity/model.py @@ -75,7 +75,8 @@ def __init__( application as a batch job """ super().__init__(name, str(path), run_settings) - self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] + self.exe = [expand_exe_path(exe)] + # self.exe = [exe] if run_settings.container else [expand_exe_path(exe)] self.exe_args = exe_args or [] self.params = params self.params_as_args = params_as_args diff --git a/smartsim/launchable/job.py b/smartsim/launchable/job.py index 8bc7536ec1..d9038db047 100644 --- a/smartsim/launchable/job.py +++ b/smartsim/launchable/job.py @@ -24,12 +24,12 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from abc import abstractmethod from copy import deepcopy +from smartsim._core.commands.launchCommands import LaunchCommands from smartsim.entity.entity import SmartSimEntity from smartsim.launchable.basejob import BaseJob -from smartsim.settings import RunSettings +from smartsim.settings import LaunchSettings class Job(BaseJob): @@ -44,8 +44,8 @@ class Job(BaseJob): def __init__( self, entity: SmartSimEntity, - launch_settings: RunSettings, # TODO: rename to LaunchSettings - ) -> None: + launch_settings: LaunchSettings, + ): super().__init__() self._entity = deepcopy(entity) self._launch_settings = deepcopy(launch_settings) @@ -60,14 +60,14 @@ def entity(self, value): self._entity = deepcopy(value) @property - def launch_settings(self) -> RunSettings: + def launch_settings(self) -> LaunchSettings: return deepcopy(self._launch_settings) @launch_settings.setter def launch_settings(self, value): self._launch_settings = deepcopy(value) - def get_launch_steps(self) -> None: # -> LaunchCommands: + def get_launch_steps(self) -> LaunchCommands: """Return the launch steps corresponding to the internal data. """ diff --git a/smartsim/launchable/mpmdjob.py b/smartsim/launchable/mpmdjob.py index e9b238f5b6..49bf8a1b36 100644 --- a/smartsim/launchable/mpmdjob.py +++ b/smartsim/launchable/mpmdjob.py @@ -31,7 +31,7 @@ from smartsim.error.errors import SSUnsupportedError from smartsim.launchable.basejob import BaseJob from smartsim.launchable.mpmdpair import MPMDPair -from smartsim.settings.base import RunSettings +from smartsim.settings.launchSettings import LaunchSettings def _check_launcher(mpmd_pairs: t.List[MPMDPair]) -> None: @@ -40,11 +40,11 @@ def _check_launcher(mpmd_pairs: t.List[MPMDPair]) -> None: ret = None for mpmd_pair in mpmd_pairs: if flag == 1: - if ret == mpmd_pair.launch_settings.run_command: + if ret == mpmd_pair.launch_settings.launcher: flag = 0 else: raise SSUnsupportedError("MPMD pairs must all share the same launcher.") - ret = mpmd_pair.launch_settings.run_command + ret = mpmd_pair.launch_settings.launcher flag = 1 @@ -86,7 +86,7 @@ def mpmd_pair(self, value): self._mpmd_pair = deepcopy(value) def add_mpmd_pair( - self, entity: SmartSimEntity, launch_settings: RunSettings + self, entity: SmartSimEntity, launch_settings: LaunchSettings ) -> None: """ Add a mpmd pair to the mpmd job diff --git a/smartsim/launchable/mpmdpair.py b/smartsim/launchable/mpmdpair.py index 37b155cb11..2b6ce36392 100644 --- a/smartsim/launchable/mpmdpair.py +++ b/smartsim/launchable/mpmdpair.py @@ -27,14 +27,12 @@ import copy from smartsim.entity.entity import SmartSimEntity -from smartsim.settings.base import RunSettings +from smartsim.settings.launchSettings import LaunchSettings class MPMDPair: """Class to store MPMD Pairs""" - def __init__( - self, entity: SmartSimEntity, launch_settings: RunSettings - ): # TODO: rename to LaunchSettings + def __init__(self, entity: SmartSimEntity, launch_settings: LaunchSettings): self.entity = copy.deepcopy(entity) self.launch_settings = copy.deepcopy(launch_settings) diff --git a/smartsim/settings/builders/batchArgBuilder.py b/smartsim/settings/builders/batchArgBuilder.py index 117bf49e5c..ad466f2541 100644 --- a/smartsim/settings/builders/batchArgBuilder.py +++ b/smartsim/settings/builders/batchArgBuilder.py @@ -28,11 +28,12 @@ import copy import typing as t -from ..._core.utils.helpers import fmt_dict from abc import ABC, abstractmethod from smartsim.log import get_logger +from ..._core.utils.helpers import fmt_dict + logger = get_logger(__name__) @@ -104,4 +105,4 @@ def format_batch_args(self) -> t.List[str]: def __str__(self) -> str: # pragma: no-cover string = f"\nScheduler Arguments:\n{fmt_dict(self._scheduler_args)}" - return string \ No newline at end of file + return string diff --git a/smartsim/settings/builders/launchArgBuilder.py b/smartsim/settings/builders/launchArgBuilder.py index 3ada3ad7b5..bb1f389f3c 100644 --- a/smartsim/settings/builders/launchArgBuilder.py +++ b/smartsim/settings/builders/launchArgBuilder.py @@ -31,6 +31,7 @@ from abc import ABC, abstractmethod from smartsim.log import get_logger + from ..._core.utils.helpers import fmt_dict logger = get_logger(__name__) @@ -67,11 +68,9 @@ def format_comma_sep_env_vars( self, env_vars: t.Dict[str, t.Optional[str]] ) -> t.Union[t.Tuple[str, t.List[str]], None]: """Build environment variable string for Slurm - Slurm takes exports in comma separated lists the list starts with all as to not disturb the rest of the environment for more information on this, see the slurm documentation for srun - :returns: the formatted string of environment variables """ logger.warning( @@ -83,7 +82,6 @@ def format_env_vars( self, env_vars: t.Dict[str, t.Optional[str]] ) -> t.Union[t.List[str], None]: """Build bash compatible environment variable string for Slurm - :returns: the formatted string of environment variables """ logger.warning(f"format_env_vars() not supported for {self.launcher_str()}.") @@ -91,4 +89,4 @@ def format_env_vars( def __str__(self) -> str: # pragma: no-cover string = f"\nLaunch Arguments:\n{fmt_dict(self._launch_args)}" - return string \ No newline at end of file + return string diff --git a/smartsim/settings/launchCommand.py b/smartsim/settings/launchCommand.py index 6f4e1aae8a..491f01d867 100644 --- a/smartsim/settings/launchCommand.py +++ b/smartsim/settings/launchCommand.py @@ -26,6 +26,7 @@ from enum import Enum + class LauncherType(Enum): """Launchers that are supported by SmartSim. diff --git a/smartsim/settings/launchSettings.py b/smartsim/settings/launchSettings.py index c29240228b..a9e5e81035 100644 --- a/smartsim/settings/launchSettings.py +++ b/smartsim/settings/launchSettings.py @@ -128,7 +128,9 @@ def update_env(self, env_vars: t.Dict[str, str | None]) -> None: if not isinstance(env, str): raise TypeError(f"The key '{env}' of env_vars should be of type str") if not isinstance(val, (str, type(None))): - raise TypeError(f"The value '{val}' of env_vars should be of type str or None") + raise TypeError( + f"The value '{val}' of env_vars should be of type str or None" + ) self._env_vars.update(env_vars) def format_env_vars(self) -> t.Union[t.List[str], None]: diff --git a/smartsim/wlm/slurm.py b/smartsim/wlm/slurm.py index 8e2b4f2b02..d8bdd630df 100644 --- a/smartsim/wlm/slurm.py +++ b/smartsim/wlm/slurm.py @@ -42,7 +42,7 @@ # from ..settings.slurmSettings import fmt_walltime # Mock function -def fmt_walltime() -> None: ... +def fmt_walltime(hours: int, minutes: int, seconds: int) -> str: ... # type: ignore[...] logger = get_logger(__name__) diff --git a/tests/temp_tests/test_colocatedJobGroup.py b/tests/temp_tests/test_colocatedJobGroup.py index 3bd49dad74..bd8ad7d6db 100644 --- a/tests/temp_tests/test_colocatedJobGroup.py +++ b/tests/temp_tests/test_colocatedJobGroup.py @@ -2,12 +2,12 @@ from smartsim.launchable.basejob import BaseJob from smartsim.launchable.colocatedJobGroup import ColocatedJobGroup from smartsim.launchable.job import Job -from smartsim.settings.base import RunSettings +from smartsim.settings import LaunchSettings # TODO replace with LaunchSettings -app_1 = Application("app_1", "python", run_settings=RunSettings()) -app_2 = Application("app_2", "python", run_settings=RunSettings()) -app_3 = Application("app_3", "python", run_settings=RunSettings()) +app_1 = Application("app_1", "python", run_settings=LaunchSettings("slurm")) +app_2 = Application("app_2", "python", run_settings=LaunchSettings("slurm")) +app_3 = Application("app_3", "python", run_settings=LaunchSettings("slurm")) def test_create_ColocatedJobGroup(): @@ -17,18 +17,18 @@ def test_create_ColocatedJobGroup(): def test_getitem_ColocatedJobGroup(): - job_1 = Job(app_1, RunSettings()) - job_2 = Job(app_2, RunSettings()) + job_1 = Job(app_1, LaunchSettings("slurm")) + job_2 = Job(app_2, LaunchSettings("slurm")) job_group = ColocatedJobGroup([job_1, job_2]) get_value = job_group[0].entity.name assert get_value == job_1.entity.name def test_setitem_JobGroup(): - job_1 = Job(app_1, RunSettings()) - job_2 = Job(app_2, RunSettings()) + job_1 = Job(app_1, LaunchSettings("slurm")) + job_2 = Job(app_2, LaunchSettings("slurm")) job_group = ColocatedJobGroup([job_1, job_2]) - job_3 = Job(app_3, RunSettings()) + job_3 = Job(app_3, LaunchSettings("slurm")) job_group[1] = job_3 assert len(job_group) == 2 get_value = job_group[1].entity.name diff --git a/tests/temp_tests/test_jobGroup.py b/tests/temp_tests/test_jobGroup.py index a5dd96d75f..2139b46e2d 100644 --- a/tests/temp_tests/test_jobGroup.py +++ b/tests/temp_tests/test_jobGroup.py @@ -2,12 +2,12 @@ from smartsim.launchable.basejob import BaseJob from smartsim.launchable.job import Job from smartsim.launchable.jobGroup import JobGroup -from smartsim.settings.base import RunSettings +from smartsim.settings.launchSettings import LaunchSettings # TODO replace with LaunchSettings -app_1 = Application("app_1", "python", RunSettings()) -app_2 = Application("app_2", "python", RunSettings()) -app_3 = Application("app_3", "python", RunSettings()) +app_1 = Application("app_1", "python", LaunchSettings("slurm")) +app_2 = Application("app_2", "python", LaunchSettings("slurm")) +app_3 = Application("app_3", "python", LaunchSettings("slurm")) def test_create_JobGroup(): @@ -17,18 +17,18 @@ def test_create_JobGroup(): def test_getitem_JobGroup(): - job_1 = Job(app_1, RunSettings()) - job_2 = Job(app_2, RunSettings()) + job_1 = Job(app_1, LaunchSettings("slurm")) + job_2 = Job(app_2, LaunchSettings("slurm")) job_group = JobGroup([job_1, job_2]) get_value = job_group[0].entity.name assert get_value == job_1.entity.name def test_setitem_JobGroup(): - job_1 = Job(app_1, RunSettings()) - job_2 = Job(app_2, RunSettings()) + job_1 = Job(app_1, LaunchSettings("slurm")) + job_2 = Job(app_2, LaunchSettings("slurm")) job_group = JobGroup([job_1, job_2]) - job_3 = Job(app_3, RunSettings()) + job_3 = Job(app_3, LaunchSettings("slurm")) job_group[1] = job_3 assert len(job_group) == 2 get_value = job_group[1] diff --git a/tests/temp_tests/test_launchable.py b/tests/temp_tests/test_launchable.py index 9d31ee5561..2a77817e58 100644 --- a/tests/temp_tests/test_launchable.py +++ b/tests/temp_tests/test_launchable.py @@ -26,7 +26,7 @@ import pytest -from smartsim.database.orchestrator import FeatureStore +from smartsim.entity.ensemble import Ensemble from smartsim.entity.entity import SmartSimEntity from smartsim.entity.model import Application from smartsim.error.errors import SSUnsupportedError @@ -34,7 +34,7 @@ from smartsim.launchable.launchable import SmartSimObject from smartsim.launchable.mpmdjob import MPMDJob from smartsim.launchable.mpmdpair import MPMDPair -from smartsim.settings.base import RunSettings +from smartsim.settings import LaunchSettings # TODO replace with LaunchSettings @@ -51,9 +51,12 @@ def test_launchable_init(): def test_job_init(): entity = Application( - "test_name", run_settings=RunSettings(), exe="echo", exe_args=["spam", "eggs"] + "test_name", + run_settings=LaunchSettings("slurm"), + exe="echo", + exe_args=["spam", "eggs"], ) - job = Job(entity, RunSettings()) + job = Job(entity, LaunchSettings("slurm")) assert isinstance(job, Job) assert job.entity.name == "test_name" assert "echo" in job.entity.exe[0] @@ -63,20 +66,24 @@ def test_job_init(): def test_job_init_deepcopy(): entity = Application( - "test_name", run_settings=RunSettings(), exe="echo", exe_args=["spam", "eggs"] + "test_name", + run_settings=LaunchSettings("slurm"), + exe="echo", + exe_args=["spam", "eggs"], ) - settings = RunSettings(run_args="test") + settings = LaunchSettings("slurm") job = Job(entity, settings) - settings.run_args = "change" - assert "change" not in job.launch_settings.run_args + test = job.launch_settings.launcher + test = "test_change" + assert job.launch_settings.launcher is not test def test_add_mpmd_pair(): - entity = SmartSimEntity("test_name", "python", RunSettings()) + entity = SmartSimEntity("test_name", "python", LaunchSettings("slurm")) mpmd_job = MPMDJob() - mpmd_job.add_mpmd_pair(entity, RunSettings()) - mpmd_pair = MPMDPair(entity, RunSettings()) + mpmd_job.add_mpmd_pair(entity, LaunchSettings("slurm")) + mpmd_pair = MPMDPair(entity, LaunchSettings("slurm")) assert len(mpmd_job.mpmd_pairs) == 1 assert str(mpmd_pair.entity) == str(mpmd_job.mpmd_pairs[0].entity) @@ -86,9 +93,12 @@ def test_add_mpmd_pair(): def test_mpmdpair_init(): """Test the creation of an MPMDPair""" entity = Application( - "test_name", "echo", exe_args=["spam", "eggs"], run_settings=RunSettings() + "test_name", + "echo", + exe_args=["spam", "eggs"], + run_settings=LaunchSettings("slurm"), ) - mpmd_pair = MPMDPair(entity, RunSettings()) + mpmd_pair = MPMDPair(entity, LaunchSettings("slurm")) assert isinstance(mpmd_pair, MPMDPair) assert mpmd_pair.entity.name == "test_name" assert "echo" in mpmd_pair.entity.exe[0] @@ -99,25 +109,35 @@ def test_mpmdpair_init(): def test_mpmdpair_init_deepcopy(): """Test the creation of an MPMDPair""" entity = Application( - "test_name", "echo", run_settings=RunSettings(), exe_args=["spam", "eggs"] + "test_name", + "echo", + run_settings=LaunchSettings("slurm"), + exe_args=["spam", "eggs"], ) - settings = RunSettings(run_args="test") + settings = LaunchSettings("slurm") mpmd_pair = MPMDPair(entity, settings) - settings.run_args = "change" - assert "change" not in mpmd_pair.launch_settings.run_args + test = mpmd_pair.launch_settings.launcher + test = "change" + assert test not in mpmd_pair.launch_settings.launcher def test_check_launcher(): """Test that mpmd pairs that have the same launcher type can be added to an MPMD Job""" entity1 = Application( - "entity1", "echo", exe_args=["hello", "world"], run_settings=RunSettings() + "entity1", + "echo", + exe_args=["hello", "world"], + run_settings=LaunchSettings("slurm"), ) - launch_settings1 = RunSettings() + launch_settings1 = LaunchSettings("slurm") entity2 = Application( - "entity2", "echo", exe_args=["hello", "world"], run_settings=RunSettings() + "entity2", + "echo", + exe_args=["hello", "world"], + run_settings=LaunchSettings("slurm"), ) - launch_settings2 = RunSettings() + launch_settings2 = LaunchSettings("slurm") mpmd_pairs = [] pair1 = MPMDPair(entity1, launch_settings1) @@ -134,11 +154,11 @@ def test_add_mpmd_pair_check_launcher_error(): """Test that an error is raised when a pairs is added to an mpmd job using add_mpmd_pair that does not have the same launcher type""" mpmd_pairs = [] - entity1 = SmartSimEntity("entity1", "python", RunSettings()) - launch_settings1 = RunSettings(run_command="srun") + entity1 = SmartSimEntity("entity1", "python", LaunchSettings("slurm")) + launch_settings1 = LaunchSettings("slurm") - entity2 = SmartSimEntity("entity2", "python", RunSettings()) - launch_settings2 = RunSettings(run_command="mpirun") + entity2 = SmartSimEntity("entity2", "python", LaunchSettings("pals")) + launch_settings2 = LaunchSettings("pals") pair1 = MPMDPair(entity1, launch_settings1) mpmd_pairs.append(pair1) @@ -152,11 +172,11 @@ def test_add_mpmd_pair_check_launcher_error(): def test_add_mpmd_pair_check_entity(): """Test that mpmd pairs that have the same entity type can be added to an MPMD Job""" mpmd_pairs = [] - entity1 = Application("entity1", "python", RunSettings()) - launch_settings1 = RunSettings(run_command="srun") + entity1 = Application("entity1", "python", LaunchSettings("slurm")) + launch_settings1 = LaunchSettings("slurm") - entity2 = Application("entity2", "python", RunSettings()) - launch_settings2 = RunSettings(run_command="srun") + entity2 = Application("entity2", "python", LaunchSettings("slurm")) + launch_settings2 = LaunchSettings("slurm") pair1 = MPMDPair(entity1, launch_settings1) mpmd_pairs.append(pair1) @@ -172,11 +192,11 @@ def test_add_mpmd_pair_check_entity_error(): """Test that an error is raised when a pairs is added to an mpmd job using add_mpmd_pair that does not have the same entity type""" mpmd_pairs = [] - entity1 = Application("entity1", "python", RunSettings()) - launch_settings1 = RunSettings(run_command="srun") + entity1 = Application("entity1", "python", LaunchSettings("slurm")) + launch_settings1 = LaunchSettings("slurm") - entity2 = FeatureStore("entity2") - launch_settings2 = RunSettings(run_command="srun") + entity2 = Application("entity2", "python", LaunchSettings("pals")) + launch_settings2 = LaunchSettings("pals") pair1 = MPMDPair(entity1, launch_settings1) mpmd_pairs.append(pair1) @@ -192,11 +212,11 @@ def test_create_mpmdjob_invalid_mpmdpairs(): does not have the same launcher type""" mpmd_pairs = [] - entity1 = Application("entity1", "python", RunSettings()) - launch_settings1 = RunSettings(run_command="srun") + entity1 = Application("entity1", "python", LaunchSettings("slurm")) + launch_settings1 = LaunchSettings("slurm") - entity1 = Application("entity1", "python", RunSettings()) - launch_settings2 = RunSettings(run_command="mpirun") + entity1 = Application("entity1", "python", LaunchSettings("pals")) + launch_settings2 = LaunchSettings("pals") pair1 = MPMDPair(entity1, launch_settings1) pair2 = MPMDPair(entity1, launch_settings2) @@ -213,10 +233,10 @@ def test_create_mpmdjob_valid_mpmdpairs(): """Test that all pairs have the same entity type is enforced when creating an MPMDJob""" mpmd_pairs = [] - entity1 = Application("entity1", "python", RunSettings()) - launch_settings1 = RunSettings(run_command="srun") - entity1 = Application("entity1", "python", RunSettings()) - launch_settings2 = RunSettings(run_command="srun") + entity1 = Application("entity1", "python", LaunchSettings("slurm")) + launch_settings1 = LaunchSettings("slurm") + entity1 = Application("entity1", "python", LaunchSettings("slurm")) + launch_settings2 = LaunchSettings("slurm") pair1 = MPMDPair(entity1, launch_settings1) pair2 = MPMDPair(entity1, launch_settings2) diff --git a/tests/temp_tests/test_settings/test_common.py b/tests/temp_tests/test_settings/test_common.py index d163275904..d303aa6e2b 100644 --- a/tests/temp_tests/test_settings/test_common.py +++ b/tests/temp_tests/test_settings/test_common.py @@ -1,10 +1,12 @@ -from smartsim.settings.common import set_check_input import pytest +from smartsim.settings.common import set_check_input + + def test_check_set_raise_error(): with pytest.raises(TypeError): set_check_input(key="test", value=3) with pytest.raises(TypeError): set_check_input(key=3, value="str") with pytest.raises(TypeError): - set_check_input(key=2, value=None) \ No newline at end of file + set_check_input(key=2, value=None) From 72987b42a4fbe0237d98b8af5ebb034c1d309c67 Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Wed, 12 Jun 2024 14:35:06 -0500 Subject: [PATCH 42/43] issue when building --- smartsim/settings/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index 2670076678..1d22e4a76f 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -34,7 +34,7 @@ logger = get_logger(__name__) -def set_check_input(key: str, value: str | None) -> None: +def set_check_input(key: str, value: t.Optional[str]) -> None: if not isinstance(key, str): raise TypeError(f"Key '{key}' should be of type str") if not isinstance(value, (str, type(None))): From ffd95d74384578c5fbe6f76091ddcf20ca670f0b Mon Sep 17 00:00:00 2001 From: Amanda Richardson Date: Thu, 13 Jun 2024 13:55:27 -0500 Subject: [PATCH 43/43] matt comment --- smartsim/settings/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smartsim/settings/common.py b/smartsim/settings/common.py index 1d22e4a76f..1f6bb5170a 100644 --- a/smartsim/settings/common.py +++ b/smartsim/settings/common.py @@ -24,6 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import annotations + import typing as t from smartsim.log import get_logger