diff --git a/smartsim/_core/commands/command_list.py b/smartsim/_core/commands/command_list.py index 34743063e6..0f10208e32 100644 --- a/smartsim/_core/commands/command_list.py +++ b/smartsim/_core/commands/command_list.py @@ -34,9 +34,11 @@ class CommandList(MutableSequence[Command]): """Container for a Sequence of Command objects""" - def __init__(self, commands: t.Union[Command, t.List[Command]]): + def __init__(self, commands: t.Optional[t.Union[Command, t.List[Command]]] = None): """CommandList constructor""" - if isinstance(commands, Command): + if commands is None: + commands = [] + elif isinstance(commands, Command): commands = [commands] self._commands: t.List[Command] = list(commands) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index 618d305710..a714eff6a4 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -49,7 +49,7 @@ def _abspath(input_path: str) -> pathlib.Path: """Helper function to check that paths are absolute""" path = pathlib.Path(input_path) if not path.is_absolute(): - raise ValueError(f"path `{path}` must be absolute") + raise ValueError(f"Path `{path}` must be absolute.") return path @@ -62,6 +62,22 @@ def _make_substitution( ) +def _prepare_param_dict(param_dict: str) -> dict[str, t.Any]: + """Decode and deserialize a base64-encoded parameter dictionary. + + This function takes a base64-encoded string representation of a dictionary, + decodes it, and then deserializes it using pickle. It performs validation + to ensure the resulting object is a non-empty dictionary. + """ + decoded_dict = base64.b64decode(param_dict) + deserialized_dict = pickle.loads(decoded_dict) + if not isinstance(deserialized_dict, dict): + raise TypeError("param dict is not a valid dictionary") + if not deserialized_dict: + raise ValueError("param dictionary is empty") + return deserialized_dict + + def _replace_tags_in( item: str, substitutions: t.Sequence[Callable[[str], str]], @@ -70,6 +86,23 @@ def _replace_tags_in( return functools.reduce(lambda a, fn: fn(a), substitutions, item) +def _process_file( + substitutions: t.Sequence[Callable[[str], str]], + source: pathlib.Path, + destination: pathlib.Path, +) -> None: + """ + Process a source file by replacing tags with specified substitutions and + write the result to a destination file. + """ + # Set the lines to iterate over + with open(source, "r+", encoding="utf-8") as file_stream: + lines = [_replace_tags_in(line, substitutions) for line in file_stream] + # write configured file to destination specified + with open(destination, "w+", encoding="utf-8") as file_stream: + file_stream.writelines(lines) + + def move(parsed_args: argparse.Namespace) -> None: """Move a source file or directory to another location. If dest is an existing directory or a symlink to a directory, then the srouce will @@ -155,9 +188,9 @@ def symlink(parsed_args: argparse.Namespace) -> None: def configure(parsed_args: argparse.Namespace) -> None: """Set, search and replace the tagged parameters for the - configure operation within tagged files attached to an entity. + configure_file operation within tagged files attached to an entity. - User-formatted files can be attached using the `configure` argument. + User-formatted files can be attached using the `configure_file` argument. These files will be modified during ``Application`` generation to replace tagged sections in the user-formatted files with values from the `params` initializer argument used during ``Application`` creation: @@ -166,39 +199,38 @@ def configure(parsed_args: argparse.Namespace) -> None: .. highlight:: bash .. code-block:: bash python -m smartsim._core.entrypoints.file_operations \ - configure /absolute/file/source/pat /absolute/file/dest/path \ + configure_file /absolute/file/source/path /absolute/file/dest/path \ tag_deliminator param_dict /absolute/file/source/path: The tagged files the search and replace operations to be performed upon /absolute/file/dest/path: The destination for configured files to be written to. - tag_delimiter: tag for the configure operation to search for, defaults to + tag_delimiter: tag for the configure_file operation to search for, defaults to semi-colon e.g. ";" param_dict: A dict of parameter names and values set for the file """ tag_delimiter = parsed_args.tag_delimiter - - decoded_dict = base64.b64decode(parsed_args.param_dict) - param_dict = pickle.loads(decoded_dict) - - if not param_dict: - raise ValueError("param dictionary is empty") - if not isinstance(param_dict, dict): - raise TypeError("param dict is not a valid dictionary") + param_dict = _prepare_param_dict(parsed_args.param_dict) substitutions = tuple( _make_substitution(k, v, tag_delimiter) for k, v in param_dict.items() ) - - # Set the lines to iterate over - with open(parsed_args.source, "r+", encoding="utf-8") as file_stream: - lines = [_replace_tags_in(line, substitutions) for line in file_stream] - - # write configured file to destination specified - with open(parsed_args.dest, "w+", encoding="utf-8") as file_stream: - file_stream.writelines(lines) + if parsed_args.source.is_dir(): + for dirpath, _, filenames in os.walk(parsed_args.source): + new_dir_dest = dirpath.replace( + str(parsed_args.source), str(parsed_args.dest), 1 + ) + os.makedirs(new_dir_dest, exist_ok=True) + for file_name in filenames: + src_file = os.path.join(dirpath, file_name) + dst_file = os.path.join(new_dir_dest, file_name) + print(type(substitutions)) + _process_file(substitutions, src_file, dst_file) + else: + dst_file = parsed_args.dest / os.path.basename(parsed_args.source) + _process_file(substitutions, parsed_args.source, dst_file) def get_parser() -> argparse.ArgumentParser: diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 801af116ce..6d31fe2ce8 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -28,206 +28,284 @@ import os import pathlib import pickle -import shutil import subprocess import sys +import time import typing as t +from collections import namedtuple from datetime import datetime -from os import mkdir, path -from os.path import join -from ...entity import Application, TaggedFilesHierarchy from ...entity.files import EntityFiles from ...launchable import Job from ...log import get_logger +from ..commands import Command, CommandList logger = get_logger(__name__) logger.propagate = False +@t.runtime_checkable +class _GenerableProtocol(t.Protocol): + """Ensures functions using job.entity continue if attrs file and params are supported.""" + + files: t.Union[EntityFiles, None] + file_parameters: t.Mapping[str, str] + + +Job_Path = namedtuple("Job_Path", ["run_path", "out_path", "err_path"]) +"""Paths related to the Job's execution.""" + + class Generator: - """The primary job of the Generator is to create the directory and file structure - for a SmartSim Job. The Generator is also responsible for writing and configuring - files into the Job directory. - """ + """The primary responsibility of the Generator class is to create the directory structure + for a SmartSim Job and to build and execute file operation commands.""" + + run_directory = "run" + """The name of the directory where run-related files are stored.""" + log_directory = "log" + """The name of the directory where log files are stored.""" def __init__(self, root: pathlib.Path) -> None: """Initialize a Generator object - The class handles symlinking, copying, and configuration of files - associated with a Jobs entity. Additionally, it writes entity parameters - used for the specific run into the "smartsim_params.txt" settings file within - the Jobs log folder. + The Generator class constructs a Job's directory structure, including: + + - The run and log directories + - Output and error files + - The "smartsim_params.txt" settings file + + Additionally, it manages symlinking, copying, and configuring files associated + with a Job's entity. + + :param root: Job base path """ self.root = root """The root path under which to generate files""" - def _generate_job_root(self, job: Job, job_index: int) -> pathlib.Path: - """Generates the root directory for a specific job instance. + def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: + """Build and return a Job's base directory. The path is created by combining the + root directory with the Job type (derived from the class name), + the name attribute of the Job, and an index to differentiate between multiple + Job runs. - :param job: The Job instance for which the root directory is generated. - :param job_index: The index of the Job instance (used for naming). - :returns: The path to the root directory for the Job instance. + :param job: Job object + :param job_index: Job index + :returns: The built file path for the Job """ job_type = f"{job.__class__.__name__.lower()}s" job_path = self.root / f"{job_type}/{job.name}-{job_index}" return pathlib.Path(job_path) - def _generate_run_path(self, job: Job, job_index: int) -> pathlib.Path: - """Generates the path for the "run" directory within the root directory - of a specific Job instance. + def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: + """Build and return a Job's run directory. The path is formed by combining + the base directory with the `run` class-level variable, where run specifies + the name of the job's run folder. - :param job (Job): The Job instance for which the path is generated. - :param job_index (int): The index of the Job instance (used for naming). - :returns: The path to the "run" directory for the Job instance. + :param job: Job object + :param job_index: Job index + :returns: The built file path for the Job run folder """ - path = self._generate_job_root(job, job_index) / "run" - path.mkdir(exist_ok=False, parents=True) + path = self._build_job_base_path(job, job_index) / self.run_directory return pathlib.Path(path) - def _generate_log_path(self, job: Job, job_index: int) -> pathlib.Path: - """ - Generates the path for the "log" directory within the root directory of a specific Job instance. + def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path: + """Build and return a Job's log directory. The path is formed by combining + the base directory with the `log` class-level variable, where log specifies + the name of the job's log folder. - :param job: The Job instance for which the path is generated. - :param job_index: The index of the Job instance (used for naming). - :returns: The path to the "log" directory for the Job instance. + :param job: Job object + :param job_index: Job index + :returns: The built file path for the Job run folder """ - path = self._generate_job_root(job, job_index) / "log" - path.mkdir(exist_ok=False, parents=True) + path = self._build_job_base_path(job, job_index) / self.log_directory return pathlib.Path(path) @staticmethod - def _log_file(log_path: pathlib.Path) -> pathlib.Path: - """Returns the location of the file - summarizing the parameters used for the generation - of the entity. + def _build_log_file_path(log_path: pathlib.Path) -> pathlib.Path: + """Build and return an entities file summarizing the parameters + used for the generation of the entity. :param log_path: Path to log directory - :returns: Path to file with parameter settings + :returns: The built file path an entities params file """ return pathlib.Path(log_path) / "smartsim_params.txt" @staticmethod - def _output_files( - log_path: pathlib.Path, job_name: str - ) -> t.Tuple[pathlib.Path, pathlib.Path]: + def _build_out_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path: + """Build and return the path to the output file. The path is created by combining + the Job's log directory with the job name and appending the `.out` extension. + + :param log_path: Path to log directory + :param job_name: Name of the Job + :returns: Path to the output file + """ out_file_path = log_path / f"{job_name}.out" + return out_file_path + + @staticmethod + def _build_err_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path: + """Build and return the path to the error file. The path is created by combining + the Job's log directory with the job name and appending the `.err` extension. + + :param log_path: Path to log directory + :param job_name: Name of the Job + :returns: Path to the error file + """ err_file_path = log_path / f"{job_name}.err" - return out_file_path, err_file_path - - def generate_job( - self, job: Job, job_index: int - ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: - """Write and configure input files for a Job. - - To have files or directories present in the created Job - directory, such as datasets or input files, call - ``entity.attach_generator_files`` prior to generation. - - Tagged application files are read, checked for input variables to - configure, and written. Input variables to configure are - specified with a tag within the input file itself. - The default tag is surrounding an input value with semicolons. - e.g. ``THERMO=;90;`` - - :param job: The job instance to write and configure files for. - :param job_path: The path to the "run" directory for the job instance. - :param log_path: The path to the "log" directory for the job instance. + return err_file_path + + def generate_job(self, job: Job, job_index: int) -> Job_Path: + """Build and return the Job's run directory, error file and out file. + + This method creates the Job's run and log directories, generates the + `smartsim_params.txt` file to log parameters used for the Job, and sets + up the output and error files for Job execution information. If files are + attached to the Job's entity, it builds file operation commands and executes + them. + + :param job: Job object + :param job_index: Job index + :return: Job's run directory, error file and out file. """ - # Generate ../job_name/run directory - job_path = self._generate_run_path(job, job_index) - # Generate ../job_name/log directory - log_path = self._generate_log_path(job, job_index) + job_path = self._build_job_run_path(job, job_index) + log_path = self._build_job_log_path(job, job_index) - # Create and write to the parameter settings file - with open(self._log_file(log_path), mode="w", encoding="utf-8") as log_file: + out_file = self._build_out_file_path(log_path, job.entity.name) + err_file = self._build_err_file_path(log_path, job.entity.name) + + cmd_list = self._build_commands(job, job_path, log_path) + + self._execute_commands(cmd_list) + + with open( + self._build_log_file_path(log_path), mode="w", encoding="utf-8" + ) as log_file: dt_string = datetime.now().strftime("%d/%m/%Y %H:%M:%S") log_file.write(f"Generation start date and time: {dt_string}\n") - # Create output files - out_file, err_file = self._output_files(log_path, job.entity.name) + return Job_Path(job_path, out_file, err_file) + + @classmethod + def _build_commands( + cls, job: Job, job_path: pathlib.Path, log_path: pathlib.Path + ) -> CommandList: + """Build file operation commands for a Job's entity. + + This method constructs commands for copying, symlinking, and writing tagged files + associated with the Job's entity. This method builds the constructs the commands to + generate the Job's run and log directory. It aggregates these commands into a CommandList + to return. + + :param job: Job object + :param job_path: The file path for the Job run folder + :return: A CommandList containing the file operation commands + """ + cmd_list = CommandList() + cmd_list.commands.append(cls._mkdir_file(job_path)) + cmd_list.commands.append(cls._mkdir_file(log_path)) + entity = job.entity + if isinstance(entity, _GenerableProtocol): + helpers: t.List[ + t.Callable[ + [t.Union[EntityFiles, None], pathlib.Path], + t.Union[CommandList, None], + ] + ] = [ + cls._copy_files, + cls._symlink_files, + lambda files, path: cls._write_tagged_files( + files, entity.file_parameters, path + ), + ] - # Perform file system operations on attached files - self._build_operations(job, job_path) + for method in helpers: + return_cmd_list = method(entity.files, job_path) + if return_cmd_list: + cmd_list.commands.extend(return_cmd_list.commands) - return job_path, out_file, err_file + return cmd_list @classmethod - def _build_operations(cls, job: Job, job_path: pathlib.Path) -> None: - """This method orchestrates file system ops for the attached SmartSim entity. - It processes three types of file system operations: to_copy, to_symlink, and to_configure. - For each type, it calls the corresponding private methods that open a subprocess - to complete each task. - - :param job: The Job to perform file ops on attached entity files - :param job_path: Path to the Jobs run directory + def _execute_commands(cls, cmd_list: CommandList) -> None: + """Execute a list of commands using subprocess. + + This helper function iterates through each command in the provided CommandList + and executes them using the subprocess module. + + :param cmd_list: A CommandList object containing the commands to be executed """ - app = t.cast(Application, job.entity) - cls._copy_files(app.files, job_path) - cls._symlink_files(app.files, job_path) - cls._write_tagged_files(app.files, app.file_parameters, job_path) + for cmd in cmd_list: + subprocess.run(cmd.command) @staticmethod - def _copy_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: - """Perform copy file sys operations on a list of files. + def _mkdir_file(file_path: pathlib.Path) -> Command: + cmd = Command(["mkdir", "-p", str(file_path)]) + return cmd - :param app: The Application attached to the Job - :param dest: Path to the Jobs run directory + @staticmethod + def _copy_files( + files: t.Union[EntityFiles, None], dest: pathlib.Path + ) -> t.Optional[CommandList]: + """Build command to copy files/directories from specified paths to a destination directory. + + This method creates commands to copy files/directories from the source paths provided in the + `files` parameter to the specified destination directory. If the source is a directory, + it copies the directory while allowing existing directories to remain intact. + + :param files: An EntityFiles object containing the paths to copy, or None. + :param dest: The destination path to the Job's run directory. + :return: A CommandList containing the copy commands, or None if no files are provided. """ - # Return if no files are attached if files is None: - return + return None + cmd_list = CommandList() for src in files.copy: + cmd = Command( + [ + sys.executable, + "-m", + "smartsim._core.entrypoints.file_operations", + "copy", + src, + ] + ) + destination = str(dest) if os.path.isdir(src): - # Remove basename of source base_source_name = os.path.basename(src) - # Attach source basename to destination - new_dst_path = os.path.join(dest, base_source_name) - # Copy source contents to new destination path - subprocess.run( - args=[ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "copy", - src, - new_dst_path, - "--dirs_exist_ok", - ] - ) + destination = os.path.join(dest, base_source_name) + cmd.append(str(destination)) + cmd.append("--dirs_exist_ok") else: - subprocess.run( - args=[ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "copy", - src, - dest, - ] - ) + cmd.append(str(dest)) + cmd_list.commands.append(cmd) + return cmd_list @staticmethod - def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> None: - """Perform symlink file sys operations on a list of files. - - :param app: The Application attached to the Job - :param dest: Path to the Jobs run directory + def _symlink_files( + files: t.Union[EntityFiles, None], dest: pathlib.Path + ) -> t.Optional[CommandList]: + """Build command to symlink files/directories from specified paths to a destination directory. + + This method creates commands to symlink files/directories from the source paths provided in the + `files` parameter to the specified destination directory. If the source is a directory, + it copies the directory while allowing existing directories to remain intact. + + :param files: An EntityFiles object containing the paths to symlink, or None. + :param dest: The destination path to the Job's run directory. + :return: A CommandList containing the symlink commands, or None if no files are provided. """ - # Return if no files are attached if files is None: - return + return None + cmd_list = CommandList() for src in files.link: # Normalize the path to remove trailing slashes normalized_path = os.path.normpath(src) # Get the parent directory (last folder) parent_dir = os.path.basename(normalized_path) - # Create destination new_dest = os.path.join(str(dest), parent_dir) - subprocess.run( - args=[ + cmd = Command( + [ sys.executable, "-m", "smartsim._core.entrypoints.file_operations", @@ -236,108 +314,45 @@ def _symlink_files(files: t.Union[EntityFiles, None], dest: pathlib.Path) -> Non new_dest, ] ) + cmd_list.append(cmd) + return cmd_list @staticmethod def _write_tagged_files( files: t.Union[EntityFiles, None], params: t.Mapping[str, str], dest: pathlib.Path, - ) -> None: - """Read, configure and write the tagged input files for - a Job instance. This function specifically deals with the tagged - files attached to an entity. + ) -> t.Optional[CommandList]: + """Build command to configure files/directories from specified paths to a destination directory. + + This method processes tagged files by reading their configurations, + serializing the provided parameters, and generating commands to + write these configurations to the destination directory. - :param app: The Application attached to the Job - :param dest: Path to the Jobs run directory + :param files: An EntityFiles object containing the paths to configure, or None. + :param params: A dictionary of params + :param dest: The destination path to the Job's run directory. + :return: A CommandList containing the configuration commands, or None if no files are provided. """ - # Return if no files are attached if files is None: - return + return None + cmd_list = CommandList() if files.tagged: - to_write = [] - - def _build_tagged_files(tagged: TaggedFilesHierarchy) -> None: - """Using a TaggedFileHierarchy, reproduce the tagged file - directory structure - - :param tagged: a TaggedFileHierarchy to be built as a - directory structure - """ - for file in tagged.files: - dst_path = path.join(dest, tagged.base, path.basename(file)) - shutil.copyfile(file, dst_path) - to_write.append(dst_path) - - for tagged_dir in tagged.dirs: - mkdir(path.join(dest, tagged.base, path.basename(tagged_dir.base))) - _build_tagged_files(tagged_dir) - - if files.tagged_hierarchy: - _build_tagged_files(files.tagged_hierarchy) - - # Pickle the dictionary + tag_delimiter = ";" pickled_dict = pickle.dumps(params) - # Default tag delimiter - tag = ";" - # Encode the pickled dictionary with Base64 encoded_dict = base64.b64encode(pickled_dict).decode("ascii") - for dest_path in to_write: - subprocess.run( - args=[ + for path in files.tagged: + cmd = Command( + [ sys.executable, "-m", "smartsim._core.entrypoints.file_operations", "configure", - dest_path, - dest_path, - tag, + path, + str(dest), + tag_delimiter, encoded_dict, ] ) - - # TODO address in ticket 723 - # self._log_params(entity, files_to_params) - - # TODO to be refactored in ticket 723 - # def _log_params( - # self, entity: Application, files_to_params: t.Dict[str, t.Dict[str, str]] - # ) -> None: - # """Log which files were modified during generation - - # and what values were set to the parameters - - # :param entity: the application being generated - # :param files_to_params: a dict connecting each file to its parameter settings - # """ - # used_params: t.Dict[str, str] = {} - # file_to_tables: t.Dict[str, str] = {} - # for file, params in files_to_params.items(): - # used_params.update(params) - # table = tabulate(params.items(), headers=["Name", "Value"]) - # file_to_tables[relpath(file, self.gen_path)] = table - - # if used_params: - # used_params_str = ", ".join( - # [f"{name}={value}" for name, value in used_params.items()] - # ) - # logger.log( - # level=self.log_level, - # msg=f"Configured application {entity.name} with params {used_params_str}", - # ) - # file_table = tabulate( - # file_to_tables.items(), - # headers=["File name", "Parameters"], - # ) - # log_entry = f"Application name: {entity.name}\n{file_table}\n\n" - # with open(self.log_file, mode="a", encoding="utf-8") as logfile: - # logfile.write(log_entry) - # with open( - # join(entity.path, "smartsim_params.txt"), mode="w", encoding="utf-8" - # ) as local_logfile: - # local_logfile.write(log_entry) - - # else: - # logger.log( - # level=self.log_level, - # msg=f"Configured application {entity.name} with no parameters", - # ) + cmd_list.commands.append(cmd) + return cmd_list diff --git a/smartsim/entity/__init__.py b/smartsim/entity/__init__.py index a12d737bb3..4f4c256289 100644 --- a/smartsim/entity/__init__.py +++ b/smartsim/entity/__init__.py @@ -28,4 +28,3 @@ from .dbnode import FSNode from .dbobject import * from .entity import SmartSimEntity, TelemetryConfiguration -from .files import TaggedFilesHierarchy diff --git a/smartsim/entity/files.py b/smartsim/entity/files.py index 9ec86a68b5..08143fbfc2 100644 --- a/smartsim/entity/files.py +++ b/smartsim/entity/files.py @@ -23,7 +23,6 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os import typing as t from os import path @@ -66,7 +65,6 @@ def __init__( self.tagged = tagged or [] self.copy = copy or [] self.link = symlink or [] - self.tagged_hierarchy = None self._check_files() def _check_files(self) -> None: @@ -82,10 +80,6 @@ def _check_files(self) -> None: self.copy = self._type_check_files(self.copy, "Copyable") self.link = self._type_check_files(self.link, "Symlink") - self.tagged_hierarchy = TaggedFilesHierarchy.from_list_paths( - self.tagged, dir_contents_to_base=True - ) - for i, value in enumerate(self.copy): self.copy[i] = self._check_path(value) @@ -147,142 +141,3 @@ def __str__(self) -> str: return "No file attached to this entity." return tabulate(values, headers=["Strategy", "Files"], tablefmt="grid") - - -class TaggedFilesHierarchy: - """The TaggedFilesHierarchy represents a directory - containing potentially tagged files and subdirectories. - - TaggedFilesHierarchy.base is the directory path from - the the root of the generated file structure - - TaggedFilesHierarchy.files is a collection of paths to - files that need to be copied to directory that the - TaggedFilesHierarchy represents - - TaggedFilesHierarchy.dirs is a collection of child - TaggedFilesHierarchy, each representing a subdirectory - that needs to generated - - By performing a depth first search over the entire - hierarchy starting at the root directory structure, the - tagged file directory structure can be replicated - """ - - def __init__(self, parent: t.Optional[t.Any] = None, subdir_name: str = "") -> None: - """Initialize a TaggedFilesHierarchy - - :param parent: The parent hierarchy of the new hierarchy, - must be None if creating a root hierarchy, - must be provided if creating a subhierachy - :param subdir_name: Name of subdirectory representd by the new hierarchy, - must be "" if creating a root hierarchy, - must be any valid dir name if subhierarchy, - invalid names are ".", ".." or contain path seperators - :raises ValueError: if given a subdir_name without a parent, - if given a parent without a subdir_name, - or if the subdir_name is invalid - """ - if parent is None and subdir_name: - raise ValueError( - "TaggedFilesHierarchies should not have a subdirectory name without a" - + " parent" - ) - if parent is not None and not subdir_name: - raise ValueError( - "Child TaggedFilesHierarchies must have a subdirectory name" - ) - if subdir_name in {".", ".."} or path.sep in subdir_name: - raise ValueError( - "Child TaggedFilesHierarchies subdirectory names must not contain" - + " path seperators or be reserved dirs '.' or '..'" - ) - - if parent: - parent.dirs.add(self) - - self._base: str = path.join(parent.base, subdir_name) if parent else "" - self.parent: t.Any = parent - self.files: t.Set[str] = set() - self.dirs: t.Set[TaggedFilesHierarchy] = set() - - @property - def base(self) -> str: - """Property to ensure that self.base is read-only""" - return self._base - - @classmethod - def from_list_paths( - cls, path_list: t.List[str], dir_contents_to_base: bool = False - ) -> t.Any: - """Given a list of absolute paths to files and dirs, create and return - a TaggedFilesHierarchy instance representing the file hierarchy of - tagged files. All files in the path list will be placed in the base of - the file hierarchy. - - :param path_list: list of absolute paths to tagged files or dirs - containing tagged files - :param dir_contents_to_base: When a top level dir is encountered, if - this value is truthy, files in the dir are - put into the base hierarchy level. - Otherwise, a new sub level is created for - the dir - :return: A built tagged file hierarchy for the given files - """ - tagged_file_hierarchy = cls() - if dir_contents_to_base: - new_paths = [] - for tagged_path in path_list: - if os.path.isdir(tagged_path): - new_paths += [ - os.path.join(tagged_path, file) - for file in os.listdir(tagged_path) - ] - else: - new_paths.append(tagged_path) - path_list = new_paths - tagged_file_hierarchy._add_paths(path_list) - return tagged_file_hierarchy - - def _add_file(self, file: str) -> None: - """Add a file to the current level in the file hierarchy - - :param file: absoute path to a file to add to the hierarchy - """ - self.files.add(file) - - def _add_dir(self, dir_path: str) -> None: - """Add a dir contianing tagged files by creating a new sub level in the - tagged file hierarchy. All paths within the directroy are added to the - the new level sub level tagged file hierarchy - - :param dir: absoute path to a dir to add to the hierarchy - """ - tagged_file_hierarchy = TaggedFilesHierarchy(self, path.basename(dir_path)) - # pylint: disable-next=protected-access - tagged_file_hierarchy._add_paths( - [path.join(dir_path, file) for file in os.listdir(dir_path)] - ) - - def _add_paths(self, paths: t.List[str]) -> None: - """Takes a list of paths and iterates over it, determining if each - path is to a file or a dir and then appropriatly adding it to the - TaggedFilesHierarchy. - - :param paths: list of paths to files or dirs to add to the hierarchy - :raises ValueError: if link to dir is found - :raises FileNotFoundError: if path does not exist - """ - for candidate in paths: - candidate = os.path.abspath(candidate) - if os.path.isdir(candidate): - if os.path.islink(candidate): - raise ValueError( - "Tagged directories and thier subdirectories cannot be links" - + " to prevent circular directory structures" - ) - self._add_dir(candidate) - elif os.path.isfile(candidate): - self._add_file(candidate) - else: - raise FileNotFoundError(f"File or Directory {candidate} not found") diff --git a/smartsim/experiment.py b/smartsim/experiment.py index fef0464758..77ad021def 100644 --- a/smartsim/experiment.py +++ b/smartsim/experiment.py @@ -38,12 +38,14 @@ from smartsim._core import dispatch from smartsim._core.config import CONFIG from smartsim._core.control import interval as _interval +from smartsim._core.control import preview_renderer from smartsim._core.control.launch_history import LaunchHistory as _LaunchHistory from smartsim._core.utils import helpers as _helpers from smartsim.error import errors from smartsim.status import TERMINAL_STATUSES, InvalidJobStatus, JobStatus -from ._core import Generator, Manifest, preview_renderer +from ._core import Generator, Manifest +from ._core.generation.generator import Job_Path from .entity import TelemetryConfiguration from .error import SmartSimError from .log import ctx_exp_path, get_logger, method_contextualizer @@ -204,8 +206,10 @@ def execute_dispatch(generator: Generator, job: Job, idx: int) -> LaunchedJobID: for_experiment=self, with_arguments=args ) # Generate the job directory and return the generated job path - job_execution_path, out, err = self._generate(generator, job, idx) - id_ = launch_config.start(exe, job_execution_path, env, out, err) + job_paths = self._generate(generator, job, idx) + id_ = launch_config.start( + exe, job_paths.run_path, env, job_paths.out_path, job_paths.err_path + ) # Save the underlying launcher instance and launched job id. That # way we do not need to spin up a launcher instance for each # individual job, and the experiment can monitor job statuses. @@ -327,9 +331,7 @@ def is_finished( return final @_contextualize - def _generate( - self, generator: Generator, job: Job, job_index: int - ) -> t.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: + def _generate(self, generator: Generator, job: Job, job_index: int) -> Job_Path: """Generate the directory structure and files for a ``Job`` If files or directories are attached to an ``Application`` object @@ -341,12 +343,12 @@ def _generate( run and log directory. :param job: The Job instance for which the output is generated. :param job_index: The index of the Job instance (used for naming). - :returns: The path to the generated output for the Job instance. + :returns: The paths to the generated output for the Job instance. :raises: A SmartSimError if an error occurs during the generation process. """ try: - job_path, out, err = generator.generate_job(job, job_index) - return (job_path, out, err) + job_paths = generator.generate_job(job, job_index) + return job_paths except SmartSimError as e: logger.error(e) raise diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 73657801d5..2157a2b96b 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -40,6 +40,7 @@ from smartsim._core import dispatch from smartsim._core.control.interval import SynchronousTimeInterval from smartsim._core.control.launch_history import LaunchHistory +from smartsim._core.generation.generator import Job_Path from smartsim._core.utils.launcher import LauncherProtocol, create_job_id from smartsim.entity import entity from smartsim.error import errors @@ -62,7 +63,9 @@ def experiment(monkeypatch, test_dir, dispatcher): monkeypatch.setattr( exp, "_generate", - lambda gen, job, idx: ("/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt"), + lambda generator, job, idx: Job_Path( + "/tmp/job", "/tmp/job/out.txt", "/tmp/job/err.txt" + ), ) yield exp diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py index 564399fd0c..327eb74286 100644 --- a/tests/test_file_operations.py +++ b/tests/test_file_operations.py @@ -30,7 +30,6 @@ import os import pathlib import pickle -import shutil from glob import glob from os import path as osp @@ -42,6 +41,10 @@ pytestmark = pytest.mark.group_a +def get_gen_file(fileutils, filename): + return fileutils.get_test_conf_path(osp.join("generator_files", filename)) + + def test_symlink_files(test_dir): """ Test operation to symlink files @@ -496,23 +499,16 @@ def test_remove_op_not_absolute(): pytest.param({}, "ValueError", id="empty dict"), ], ) -def test_configure_op(test_dir, fileutils, param_dict, error_type): - """Test configure operation with correct parameter dictionary, empty dicitonary, and an incorrect type""" +def test_configure_file_op(test_dir, fileutils, param_dict, error_type): + """Test configure file operation with correct parameter dictionary, empty dicitonary, and an incorrect type""" tag = ";" - conf_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "marked/") - ) # retrieve files to compare after test correct_path = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "correct/") ) - # copy files to test directory - shutil.copytree(conf_path, test_dir, dirs_exist_ok=True) - assert osp.isdir(test_dir) - tagged_files = sorted(glob(test_dir + "/*")) correct_files = sorted(glob(correct_path + "/*")) @@ -545,12 +541,12 @@ def test_configure_op(test_dir, fileutils, param_dict, error_type): assert filecmp.cmp(written, correct) -def test_configure_invalid_tags(fileutils, test_dir): - """Test configure operation with an invalid tag""" +def test_configure_file_invalid_tags(fileutils, test_dir): + """Test configure file operation with an invalid tag""" generator_files = pathlib.Path(fileutils.get_test_conf_path("generator_files")) tagged_file = generator_files / "easy/marked/invalidtag.txt" correct_file = generator_files / "easy/correct/invalidtag.txt" - target_file = pathlib.Path(test_dir, "target.txt") + target_file = pathlib.Path(test_dir, "invalidtag.txt") tag = ";" param_dict = {"VALID": "valid"} @@ -561,7 +557,7 @@ def test_configure_invalid_tags(fileutils, test_dir): # Encode the pickled dictionary with Base64 encoded_dict = base64.b64encode(pickled_dict).decode("ascii") parser = get_parser() - cmd = f"configure {tagged_file} {target_file} {tag} {encoded_dict}" + cmd = f"configure {tagged_file} {test_dir} {tag} {encoded_dict}" args = cmd.split() ns = parser.parse_args(args) @@ -569,9 +565,9 @@ def test_configure_invalid_tags(fileutils, test_dir): assert filecmp.cmp(correct_file, target_file) -def test_configure_not_absolute(): +def test_configure_file_not_absolute(): """Test that ValueError is raised when tagged files - given to configure op are not absolute paths + given to configure file op are not absolute paths """ tagged_file = ".." @@ -593,6 +589,83 @@ def test_configure_not_absolute(): assert "invalid _abspath value" in e.value.__context__.message +@pytest.mark.parametrize( + ["param_dict", "error_type"], + [ + pytest.param( + {"PARAM0": "param_value_1", "PARAM1": "param_value_2"}, + "None", + id="correct dict", + ), + pytest.param( + ["list", "of", "values"], + "TypeError", + id="incorrect dict", + ), + pytest.param({}, "ValueError", id="empty dict"), + ], +) +def test_configure_directory(test_dir, fileutils, param_dict, error_type): + """Test configure directory operation with correct parameter dictionary, empty dicitonary, and an incorrect type""" + tag = ";" + config = get_gen_file(fileutils, "tag_dir_template") + + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + + parser = get_parser() + cmd = f"configure {config} {test_dir} {tag} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + if error_type == "ValueError": + with pytest.raises(ValueError) as ex: + file_operations.configure(ns) + assert "param dictionary is empty" in ex.value.args[0] + elif error_type == "TypeError": + with pytest.raises(TypeError) as ex: + file_operations.configure(ns) + assert "param dict is not a valid dictionary" in ex.value.args[0] + else: + file_operations.configure(ns) + assert osp.isdir(osp.join(test_dir, "nested_0")) + assert osp.isdir(osp.join(test_dir, "nested_1")) + + with open(osp.join(test_dir, "nested_0", "tagged_0.sh")) as f: + line = f.readline() + assert line.strip() == f'echo "Hello with parameter 0 = param_value_1"' + + with open(osp.join(test_dir, "nested_1", "tagged_1.sh")) as f: + line = f.readline() + assert line.strip() == f'echo "Hello with parameter 1 = param_value_2"' + + +def test_configure_directory_not_absolute(): + """Test that ValueError is raised when tagged directories + given to configure op are not absolute paths + """ + + tagged_directory = ".." + tag = ";" + param_dict = {"5": 10} + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict) + parser = get_parser() + cmd = f"configure {tagged_directory} {tagged_directory} {tag} {encoded_dict}" + args = cmd.split() + + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message + + def test_parser_move(): """Test that the parser succeeds when receiving expected args for the move operation""" parser = get_parser() @@ -653,8 +726,38 @@ def test_parser_copy(): assert ns.dest == dest_path -def test_parser_configure_parse(): - """Test that the parser succeeds when receiving expected args for the configure operation""" +def test_parser_configure_file_parse(): + """Test that the parser succeeds when receiving expected args for the configure file operation""" + parser = get_parser() + + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") + tag_delimiter = ";" + + param_dict = { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + } + + pickled_dict = pickle.dumps(param_dict) + encoded_dict = base64.b64encode(pickled_dict) + + cmd = f"configure {src_path} {dest_path} {tag_delimiter} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + assert ns.source == src_path + assert ns.dest == dest_path + assert ns.tag_delimiter == tag_delimiter + assert ns.param_dict == str(encoded_dict) + + +def test_parser_configure_directory_parse(): + """Test that the parser succeeds when receiving expected args for the configure directory operation""" parser = get_parser() src_path = pathlib.Path("/absolute/file/src/path") diff --git a/tests/test_generator.py b/tests/test_generator.py index 8f5a02f0b6..4c25ccd05f 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,8 +1,33 @@ -import filecmp +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import itertools import os import pathlib -import random +import unittest.mock from glob import glob from os import listdir from os import path as osp @@ -10,9 +35,10 @@ import pytest from smartsim import Experiment +from smartsim._core.commands import Command, CommandList from smartsim._core.generation.generator import Generator from smartsim.builders import Ensemble -from smartsim.entity import Application +from smartsim.entity import entity from smartsim.entity.files import EntityFiles from smartsim.launchable import Job from smartsim.settings import LaunchSettings @@ -21,6 +47,9 @@ pytestmark = pytest.mark.group_a +ids = set() + + _ID_GENERATOR = (str(i) for i in itertools.count()) @@ -44,164 +73,237 @@ def get_gen_configure_dir(fileutils): @pytest.fixture -def generator_instance(test_dir) -> Generator: +def generator_instance(test_dir: str) -> Generator: """Fixture to create an instance of Generator.""" root = pathlib.Path(test_dir, "temp_id") + os.mkdir(root) yield Generator(root=root) -def test_log_file_path(generator_instance): - """Test if the log_file function returns the correct log path.""" - base_path = "/tmp" - expected_path = osp.join(base_path, "smartsim_params.txt") - assert generator_instance._log_file(base_path) == pathlib.Path(expected_path) +def get_gen_file(fileutils, filename: str): + return fileutils.get_test_conf_path(osp.join("generator_files", filename)) -def test_generate_job_directory(test_dir, wlmutils, generator_instance): - """Test Generator.generate_job""" - # Create Job - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("app_name", exe="python") - job = Job(app, launch_settings) - # Mock id - run_id = "temp_id" - # Call Generator.generate_job - job_run_path, _, _ = generator_instance.generate_job(job, 0) - assert isinstance(job_run_path, pathlib.Path) - expected_run_path = ( - pathlib.Path(test_dir) - / run_id - / f"{job.__class__.__name__.lower()}s" - / f"{app.name}-{0}" - / "run" +class EchoHelloWorldEntity(entity.SmartSimEntity): + """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" + + def __init__(self): + self.name = "entity_name" + self.files = None + self.file_parameters = None + + def as_executable_sequence(self): + return ("echo", "Hello", "World!") + + def files(): + return ["file_path"] + + +@pytest.fixture +def mock_job() -> unittest.mock.MagicMock: + """Fixture to create a mock Job.""" + job = unittest.mock.MagicMock( + **{ + "entity": EchoHelloWorldEntity(), + "name": "test_job", + "get_launch_steps": unittest.mock.MagicMock( + side_effect=lambda: NotImplementedError() + ), + }, + spec=Job, ) - assert job_run_path == expected_run_path - expected_log_path = ( - pathlib.Path(test_dir) - / run_id - / f"{job.__class__.__name__.lower()}s" - / f"{app.name}-{0}" - / "log" + yield job + + +# UNIT TESTS + + +def test_init_generator(generator_instance: Generator, test_dir: str): + """Test Generator init""" + assert generator_instance.root == pathlib.Path(test_dir) / "temp_id" + + +def test_build_job_base_path( + generator_instance: Generator, mock_job: unittest.mock.MagicMock +): + """Test Generator._build_job_base_path returns correct path""" + mock_index = 1 + root_path = generator_instance._build_job_base_path(mock_job, mock_index) + expected_path = ( + generator_instance.root + / f"{mock_job.__class__.__name__.lower()}s" + / f"{mock_job.name}-{mock_index}" ) - assert osp.isdir(expected_run_path) - assert osp.isdir(expected_log_path) - # Assert smartsim params file created - assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) - # Assert smartsim params correctly written to - with open(expected_log_path / "smartsim_params.txt", "r") as file: - content = file.read() - assert "Generation start date and time:" in content - - -def test_exp_private_generate_method(wlmutils, test_dir, generator_instance): - """Test that Job directory was created from Experiment._generate.""" - # Create Experiment - exp = Experiment(name="experiment_name", exp_path=test_dir) - # Create Job - app = Application("name", "python") - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job = Job(app, launch_settings) - # Generate Job directory - job_index = 1 - job_execution_path, _, _ = exp._generate(generator_instance, job, job_index) - # Assert Job run directory exists - assert osp.isdir(job_execution_path) - # Assert Job log directory exists - head, _ = os.path.split(job_execution_path) - expected_log_path = pathlib.Path(head) / "log" - assert osp.isdir(expected_log_path) - - -def test_generate_copy_file(generator_instance, fileutils, wlmutils): - """Test that attached copy files are copied into Job directory""" - # Create the Job and attach copy generator file - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python") - script = fileutils.get_test_conf_path("sleep.py") - app.attach_generator_files(to_copy=script) - job = Job(app, launch_settings) + assert root_path == expected_path - # Create the experiment - path, _, _ = generator_instance.generate_job(job, 1) - expected_file = pathlib.Path(path) / "sleep.py" - assert osp.isfile(expected_file) +def test_build_job_run_path( + test_dir: str, + mock_job: unittest.mock.MagicMock, + generator_instance: Generator, + monkeypatch: pytest.MonkeyPatch, +): + """Test Generator._build_job_run_path returns correct path""" + mock_index = 1 + monkeypatch.setattr( + Generator, + "_build_job_base_path", + lambda self, job, job_index: pathlib.Path(test_dir), + ) + run_path = generator_instance._build_job_run_path(mock_job, mock_index) + expected_run_path = pathlib.Path(test_dir) / "run" + assert run_path == expected_run_path -def test_generate_copy_directory(wlmutils, get_gen_copy_dir, generator_instance): - # Create the Job and attach generator file - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python") - app.attach_generator_files(to_copy=get_gen_copy_dir) - job = Job(app, launch_settings) - # Call Generator.generate_job - path, _, _ = generator_instance.generate_job(job, 1) - expected_folder = path / "to_copy_dir" - assert osp.isdir(expected_folder) +def test_build_job_log_path( + test_dir: str, + mock_job: unittest.mock.MagicMock, + generator_instance: Generator, + monkeypatch: pytest.MonkeyPatch, +): + """Test Generator._build_job_log_path returns correct path""" + mock_index = 1 + monkeypatch.setattr( + Generator, + "_build_job_base_path", + lambda self, job, job_index: pathlib.Path(test_dir), + ) + log_path = generator_instance._build_job_log_path(mock_job, mock_index) + expected_log_path = pathlib.Path(test_dir) / "log" + assert log_path == expected_log_path + +def test_build_log_file_path(test_dir: str, generator_instance: Generator): + """Test Generator._build_log_file_path returns correct path""" + expected_path = pathlib.Path(test_dir) / "smartsim_params.txt" + assert generator_instance._build_log_file_path(test_dir) == expected_path -def test_generate_symlink_directory(wlmutils, generator_instance, get_gen_symlink_dir): - # Create the Job and attach generator file - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python") - # Attach directory to Application - app.attach_generator_files(to_symlink=get_gen_symlink_dir) - # Create Job - job = Job(app, launch_settings) - - # Call Generator.generate_job - path, _, _ = generator_instance.generate_job(job, 1) - expected_folder = path / "to_symlink_dir" - assert osp.isdir(expected_folder) - assert expected_folder.is_symlink() - assert os.fspath(expected_folder.resolve()) == osp.realpath(get_gen_symlink_dir) - # Combine symlinked file list and original file list for comparison - for written, correct in itertools.zip_longest( - listdir(get_gen_symlink_dir), listdir(expected_folder) + +def test_build_out_file_path( + test_dir: str, generator_instance: Generator, mock_job: unittest.mock.MagicMock +): + """Test Generator._build_out_file_path returns out path""" + out_file_path = generator_instance._build_out_file_path( + pathlib.Path(test_dir), mock_job.name + ) + assert out_file_path == pathlib.Path(test_dir) / f"{mock_job.name}.out" + + +def test_build_err_file_path( + test_dir: str, generator_instance: Generator, mock_job: unittest.mock.MagicMock +): + """Test Generator._build_err_file_path returns err path""" + err_file_path = generator_instance._build_err_file_path( + pathlib.Path(test_dir), mock_job.name + ) + assert err_file_path == pathlib.Path(test_dir) / f"{mock_job.name}.err" + + +def test_generate_job( + mock_job: unittest.mock.MagicMock, + generator_instance: Generator, +): + """Test Generator.generate_job returns correct paths""" + mock_index = 1 + job_paths = generator_instance.generate_job(mock_job, mock_index) + assert job_paths.run_path.name == Generator.run_directory + assert job_paths.out_path.name == f"{mock_job.entity.name}.out" + assert job_paths.err_path.name == f"{mock_job.entity.name}.err" + + +def test_build_commands( + mock_job: unittest.mock.MagicMock, generator_instance: Generator, test_dir: str +): + """Test Generator._build_commands calls correct helper functions""" + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._copy_files" + ) as mock_copy_files, + unittest.mock.patch( + "smartsim._core.generation.Generator._symlink_files" + ) as mock_symlink_files, + unittest.mock.patch( + "smartsim._core.generation.Generator._write_tagged_files" + ) as mock_write_tagged_files, + ): + generator_instance._build_commands( + mock_job, + pathlib.Path(test_dir) / generator_instance.run_directory, + pathlib.Path(test_dir) / generator_instance.log_directory, + ) + mock_copy_files.assert_called_once() + mock_symlink_files.assert_called_once() + mock_write_tagged_files.assert_called_once() + + +def test_execute_commands(generator_instance: Generator): + """Test Generator._execute_commands subprocess.run""" + with ( + unittest.mock.patch( + "smartsim._core.generation.generator.subprocess.run" + ) as run_process, ): - # For each pair, check if the filenames are equal - assert written == correct + cmd_list = CommandList(Command(["test", "command"])) + generator_instance._execute_commands(cmd_list) + run_process.assert_called_once() -def test_generate_symlink_file(get_gen_symlink_dir, wlmutils, generator_instance): - # Create the Job and attach generator file - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - app = Application("name", "python") - # Path of directory to symlink - symlink_dir = get_gen_symlink_dir - # Get a list of all files in the directory - symlink_files = sorted(glob(symlink_dir + "/*")) - # Attach directory to Application - app.attach_generator_files(to_symlink=symlink_files) - # Create Job - job = Job(app, launch_settings) - - # Call Generator.generate_job - path, _, _ = generator_instance.generate_job(job, 1) - expected_file = path / "mock2.txt" - assert osp.isfile(expected_file) - assert expected_file.is_symlink() - assert os.fspath(expected_file.resolve()) == osp.join( - osp.realpath(get_gen_symlink_dir), "mock2.txt" +def test_mkdir_file(generator_instance: Generator, test_dir: str): + """Test Generator._mkdir_file returns correct type and value""" + cmd = generator_instance._mkdir_file(pathlib.Path(test_dir)) + assert isinstance(cmd, Command) + assert cmd.command == ["mkdir", "-p", test_dir] + + +def test_copy_file(generator_instance: Generator, fileutils): + """Test Generator._copy_files helper function with file""" + script = fileutils.get_test_conf_path("sleep.py") + files = EntityFiles(copy=script) + cmd_list = generator_instance._copy_files(files, generator_instance.root) + assert isinstance(cmd_list, CommandList) + assert len(cmd_list) == 1 + assert str(generator_instance.root) and script in cmd_list.commands[0].command + + +def test_copy_directory(get_gen_copy_dir, generator_instance: Generator): + """Test Generator._copy_files helper function with directory""" + files = EntityFiles(copy=get_gen_copy_dir) + cmd_list = generator_instance._copy_files(files, generator_instance.root) + assert isinstance(cmd_list, CommandList) + assert len(cmd_list) == 1 + assert ( + str(generator_instance.root) + and get_gen_copy_dir in cmd_list.commands[0].command ) -def test_generate_configure(fileutils, wlmutils, generator_instance): - # Directory of files to configure +def test_symlink_file(get_gen_symlink_dir, generator_instance: Generator): + """Test Generator._symlink_files helper function with file list""" + symlink_files = sorted(glob(get_gen_symlink_dir + "/*")) + files = EntityFiles(symlink=symlink_files) + cmd_list = generator_instance._symlink_files(files, generator_instance.root) + assert isinstance(cmd_list, CommandList) + for file, cmd in zip(symlink_files, cmd_list): + assert file in cmd.command + + +def test_symlink_directory(generator_instance: Generator, get_gen_symlink_dir): + """Test Generator._symlink_files helper function with directory""" + files = EntityFiles(symlink=get_gen_symlink_dir) + cmd_list = generator_instance._symlink_files(files, generator_instance.root) + symlinked_folder = generator_instance.root / os.path.basename(get_gen_symlink_dir) + assert isinstance(cmd_list, CommandList) + assert str(symlinked_folder) in cmd_list.commands[0].command + + +def test_write_tagged_file(fileutils, generator_instance: Generator): + """Test Generator._write_tagged_files helper function with file list""" conf_path = fileutils.get_test_conf_path( osp.join("generator_files", "easy", "marked/") ) - # Retrieve a list of files for configuration tagged_files = sorted(glob(conf_path + "/*")) - # Retrieve directory of files to compare after Experiment.generate_experiment completion - correct_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "correct/") - ) - # Retrieve list of files in correctly tagged directory for comparison - correct_files = sorted(glob(correct_path + "/*")) - # Initialize a Job - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - param_dict = { + files = EntityFiles(tagged=tagged_files) + param_set = { "5": 10, "FIRST": "SECOND", "17": 20, @@ -210,54 +312,46 @@ def test_generate_configure(fileutils, wlmutils, generator_instance): "1200": "120", "VALID": "valid", } - app = Application("name_1", "python", file_parameters=param_dict) - app.attach_generator_files(to_configure=tagged_files) - job = Job(app, launch_settings) + cmd_list = generator_instance._write_tagged_files( + files=files, params=param_set, dest=generator_instance.root + ) + assert isinstance(cmd_list, CommandList) + for file, cmd in zip(tagged_files, cmd_list): + assert file in cmd.command + + +def test_write_tagged_directory(fileutils, generator_instance: Generator): + """Test Generator._write_tagged_files helper function with directory path""" + config = get_gen_file(fileutils, "tag_dir_template") + files = EntityFiles(tagged=[config]) + param_set = {"PARAM0": "param_value_1", "PARAM1": "param_value_2"} + cmd_list = generator_instance._write_tagged_files( + files=files, params=param_set, dest=generator_instance.root + ) - # Call Generator.generate_job - path, _, _ = generator_instance.generate_job(job, 0) - # Retrieve the list of configured files in the test directory - configured_files = sorted(glob(str(path) + "/*")) - # Use filecmp.cmp to check that the corresponding files are equal - for written, correct in itertools.zip_longest(configured_files, correct_files): - assert filecmp.cmp(written, correct) + assert isinstance(cmd_list, CommandList) + assert str(config) in cmd_list.commands[0].command -def test_exp_private_generate_method_ensemble(test_dir, wlmutils, generator_instance): - """Test that Job directory was created from Experiment.""" - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - for i, job in enumerate(job_list): - job_run_path, _, _ = exp._generate(generator_instance, job, i) - head, _ = os.path.split(job_run_path) - expected_log_path = pathlib.Path(head) / "log" - assert osp.isdir(job_run_path) - assert osp.isdir(pathlib.Path(expected_log_path)) +# INTEGRATED TESTS -def test_generate_ensemble_directory(wlmutils, generator_instance): - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - for i, job in enumerate(job_list): - # Call Generator.generate_job - path, _, _ = generator_instance.generate_job(job, i) - # Assert run directory created - assert osp.isdir(path) - # Assert smartsim params file created - head, _ = os.path.split(path) - expected_log_path = pathlib.Path(head) / "log" - assert osp.isdir(expected_log_path) - assert osp.isfile(osp.join(expected_log_path, "smartsim_params.txt")) - # Assert smartsim params correctly written to - with open(expected_log_path / "smartsim_params.txt", "r") as file: - content = file.read() - assert "Generation start date and time:" in content - - -def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): +def test_exp_private_generate_method( + mock_job: unittest.mock.MagicMock, test_dir: str, generator_instance: Generator +): + """Test that Experiment._generate returns expected tuple.""" + mock_index = 1 + exp = Experiment(name="experiment_name", exp_path=test_dir) + job_paths = exp._generate(generator_instance, mock_job, mock_index) + assert osp.isdir(job_paths.run_path) + assert job_paths.out_path.name == f"{mock_job.entity.name}.out" + assert job_paths.err_path.name == f"{mock_job.entity.name}.err" + + +def test_generate_ensemble_directory_start( + test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch +): + """Test that Experiment._generate returns expected tuple.""" monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), @@ -268,16 +362,19 @@ def test_generate_ensemble_directory_start(test_dir, wlmutils, monkeypatch): exp = Experiment(name="exp_name", exp_path=test_dir) exp.start(*job_list) run_dir = listdir(test_dir) - jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") - job_dir = listdir(jobs_dir) - for ensemble_dir in job_dir: - run_path = os.path.join(jobs_dir, ensemble_dir, "run") - log_path = os.path.join(jobs_dir, ensemble_dir, "log") - assert osp.isdir(run_path) - assert osp.isdir(log_path) - - -def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_dir): + jobs_dir_path = pathlib.Path(test_dir) / run_dir[0] / "jobs" + list_of_job_dirs = jobs_dir_path.iterdir() + for job in list_of_job_dirs: + run_path = jobs_dir_path / job / Generator.run_directory + assert run_path.is_dir() + log_path = jobs_dir_path / job / Generator.log_directory + assert log_path.is_dir() + ids.clear() + + +def test_generate_ensemble_copy( + test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_copy_dir +): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), @@ -290,15 +387,18 @@ def test_generate_ensemble_copy(test_dir, wlmutils, monkeypatch, get_gen_copy_di exp = Experiment(name="exp_name", exp_path=test_dir) exp.start(*job_list) run_dir = listdir(test_dir) - jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") - job_dir = listdir(jobs_dir) + jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" + job_dir = jobs_dir.iterdir() for ensemble_dir in job_dir: - copy_folder_path = os.path.join(jobs_dir, ensemble_dir, "run", "to_copy_dir") - assert osp.isdir(copy_folder_path) + copy_folder_path = ( + jobs_dir / ensemble_dir / Generator.run_directory / "to_copy_dir" + ) + assert copy_folder_path.is_dir() + ids.clear() def test_generate_ensemble_symlink( - test_dir, wlmutils, monkeypatch, get_gen_symlink_dir + test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_symlink_dir ): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", @@ -313,55 +413,58 @@ def test_generate_ensemble_symlink( launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.build_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) + _ = exp.start(*job_list) run_dir = listdir(test_dir) - jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") - job_dir = listdir(jobs_dir) + jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" + job_dir = jobs_dir.iterdir() for ensemble_dir in job_dir: sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" - assert osp.isdir(sym_file_path) + assert sym_file_path.is_dir() assert sym_file_path.is_symlink() assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) + ids.clear() def test_generate_ensemble_configure( - test_dir, wlmutils, monkeypatch, get_gen_configure_dir + test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_configure_dir ): monkeypatch.setattr( "smartsim._core.dispatch._LauncherAdapter.start", lambda launch, exe, job_execution_path, env, out, err: random_id(), ) - params = {"PARAM0": [0, 1], "PARAM1": [2, 3]} - # Retrieve a list of files for configuration + param_set = {"PARAM0": [0, 1], "PARAM1": [2, 3]} tagged_files = sorted(glob(get_gen_configure_dir + "/*")) ensemble = Ensemble( "ensemble-name", "echo", replicas=1, files=EntityFiles(tagged=tagged_files), - file_parameters=params, + file_parameters=param_set, ) launch_settings = LaunchSettings(wlmutils.get_test_launcher()) job_list = ensemble.build_jobs(launch_settings) exp = Experiment(name="exp_name", exp_path=test_dir) - id = exp.start(*job_list) + _ = exp.start(*job_list) run_dir = listdir(test_dir) - jobs_dir = os.path.join(test_dir, run_dir[0], "jobs") + jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" def _check_generated(param_0, param_1, dir): - assert osp.isdir(dir) - assert osp.isfile(osp.join(dir, "tagged_0.sh")) - assert osp.isfile(osp.join(dir, "tagged_1.sh")) + assert dir.is_dir() + tagged_0 = dir / "tagged_0.sh" + tagged_1 = dir / "tagged_1.sh" + assert tagged_0.is_file() + assert tagged_1.is_file() - with open(osp.join(dir, "tagged_0.sh")) as f: + with open(tagged_0) as f: line = f.readline() assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' - with open(osp.join(dir, "tagged_1.sh")) as f: + with open(tagged_1) as f: line = f.readline() assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' - _check_generated(0, 3, os.path.join(jobs_dir, "ensemble-name-1-1", "run")) - _check_generated(1, 2, os.path.join(jobs_dir, "ensemble-name-2-2", "run")) - _check_generated(1, 3, os.path.join(jobs_dir, "ensemble-name-3-3", "run")) - _check_generated(0, 2, os.path.join(jobs_dir, "ensemble-name-0-0", "run")) + _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run_directory) + _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run_directory) + _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory) + _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory) + ids.clear()