diff --git a/conftest.py b/conftest.py index c407681d76..895fcc9adb 100644 --- a/conftest.py +++ b/conftest.py @@ -40,6 +40,8 @@ import typing as t import uuid import warnings +from glob import glob +from os import path as osp from collections import defaultdict from dataclasses import dataclass from subprocess import run @@ -53,6 +55,8 @@ from smartsim._core.config.config import Config from smartsim._core.launcher.dragon.dragon_connector import DragonConnector from smartsim._core.launcher.dragon.dragon_launcher import DragonLauncher +from smartsim._core.generation.operations.operations import ConfigureOperation, CopyOperation, SymlinkOperation +from smartsim._core.generation.generator import Generator from smartsim._core.utils.telemetry.telemetry import JobEntity from smartsim.database import FeatureStore from smartsim.entity import Application @@ -469,6 +473,58 @@ def check_output_dir() -> None: def fsutils() -> t.Type[FSUtils]: return FSUtils +@pytest.fixture +def files(fileutils): + path_to_files = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + list_of_files_strs = glob(path_to_files + "/*") + yield [pathlib.Path(str_path) for str_path in list_of_files_strs] + + +@pytest.fixture +def directory(fileutils): + directory = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + yield [pathlib.Path(directory)] + + +@pytest.fixture(params=["files", "directory"]) +def source(request): + yield request.getfixturevalue(request.param) + + +@pytest.fixture +def mock_src(test_dir: str): + """Fixture to create a mock source path.""" + return pathlib.Path(test_dir) / pathlib.Path("mock_src") + + +@pytest.fixture +def mock_dest(): + """Fixture to create a mock destination path.""" + return pathlib.Path("mock_dest") + + +@pytest.fixture +def copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return CopyOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def symlink_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a CopyOperation object.""" + return SymlinkOperation(src=mock_src, dest=mock_dest) + + +@pytest.fixture +def configure_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Fixture to create a Configure object.""" + return ConfigureOperation( + src=mock_src, dest=mock_dest, file_parameters={"FOO": "BAR"} + ) class FSUtils: @staticmethod diff --git a/smartsim/_core/commands/command.py b/smartsim/_core/commands/command.py index 3f41f32fe9..0968759afd 100644 --- a/smartsim/_core/commands/command.py +++ b/smartsim/_core/commands/command.py @@ -35,6 +35,10 @@ class Command(MutableSequence[str]): """Basic container for command information""" def __init__(self, command: t.List[str]) -> None: + if not command: + raise TypeError("Command list cannot be empty") + if not all(isinstance(item, str) for item in command): + raise TypeError("All items in the command list must be strings") """Command constructor""" self._command = command @@ -66,7 +70,7 @@ def __setitem__( """Set the command at the specified index.""" if isinstance(idx, int): if not isinstance(value, str): - raise ValueError( + raise TypeError( "Value must be of type `str` when assigning to an index" ) self._command[idx] = deepcopy(value) @@ -74,9 +78,7 @@ def __setitem__( if not isinstance(value, list) or not all( isinstance(item, str) for item in value ): - raise ValueError( - "Value must be a list of strings when assigning to a slice" - ) + raise TypeError("Value must be a list of strings when assigning to a slice") self._command[idx] = (deepcopy(val) for val in value) def __delitem__(self, idx: t.Union[int, slice]) -> None: diff --git a/smartsim/_core/commands/command_list.py b/smartsim/_core/commands/command_list.py index 0f10208e32..fcffe42a2a 100644 --- a/smartsim/_core/commands/command_list.py +++ b/smartsim/_core/commands/command_list.py @@ -69,20 +69,20 @@ def __setitem__( """Set the Commands at the specified index.""" if isinstance(idx, int): if not isinstance(value, Command): - raise ValueError( + raise TypeError( "Value must be of type `Command` when assigning to an index" ) self._commands[idx] = deepcopy(value) return if not isinstance(value, list): - raise ValueError( + raise TypeError( "Value must be a list of Commands when assigning to a slice" ) for sublist in value: if not isinstance(sublist.command, list) or not all( isinstance(item, str) for item in sublist.command ): - raise ValueError( + raise TypeError( "Value sublists must be a list of Commands when assigning to a slice" ) self._commands[idx] = (deepcopy(val) for val in value) diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py index a714eff6a4..69d7f7565e 100644 --- a/smartsim/_core/entrypoints/file_operations.py +++ b/smartsim/_core/entrypoints/file_operations.py @@ -154,9 +154,9 @@ def copy(parsed_args: argparse.Namespace) -> None: /absolute/file/dest/path: Path to destination directory or path to destination file --dirs_exist_ok: if the flag is included, the copying operation will - continue if the destination directory and files alrady exist, + continue if the destination directory and files already exist, and will be overwritten by corresponding files. If the flag is - not includedm and the destination file already exists, a + not included and the destination file already exists, a FileExistsError will be raised """ if os.path.isdir(parsed_args.source): @@ -226,7 +226,6 @@ def configure(parsed_args: argparse.Namespace) -> None: for file_name in filenames: src_file = os.path.join(dirpath, file_name) dst_file = os.path.join(new_dir_dest, file_name) - print(type(substitutions)) _process_file(substitutions, src_file, dst_file) else: dst_file = parsed_args.dest / os.path.basename(parsed_args.source) diff --git a/smartsim/_core/generation/generator.py b/smartsim/_core/generation/generator.py index 6d31fe2ce8..1cc1670655 100644 --- a/smartsim/_core/generation/generator.py +++ b/smartsim/_core/generation/generator.py @@ -24,21 +24,23 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import base64 -import os import pathlib -import pickle import subprocess -import sys -import time import typing as t from collections import namedtuple from datetime import datetime -from ...entity.files import EntityFiles +from ...entity import entity from ...launchable import Job from ...log import get_logger from ..commands import Command, CommandList +from .operations.operations import ( + ConfigureOperation, + CopyOperation, + FileSysOperationSet, + GenerationContext, + SymlinkOperation, +) logger = get_logger(__name__) logger.propagate = False @@ -46,41 +48,44 @@ @t.runtime_checkable class _GenerableProtocol(t.Protocol): - """Ensures functions using job.entity continue if attrs file and params are supported.""" + """Protocol to ensure that an entity supports both file operations + and parameters.""" - files: t.Union[EntityFiles, None] + files: FileSysOperationSet + # TODO change when file_parameters taken off Application during Ensemble refactor ticket file_parameters: t.Mapping[str, str] Job_Path = namedtuple("Job_Path", ["run_path", "out_path", "err_path"]) -"""Paths related to the Job's execution.""" +"""Namedtuple that stores a Job's run directory, output file path, and +error file path.""" class Generator: - """The primary responsibility of the Generator class is to create the directory structure - for a SmartSim Job and to build and execute file operation commands.""" + """The Generator class creates the directory structure for a SmartSim Job by building + and executing file operation commands. + """ run_directory = "run" - """The name of the directory where run-related files are stored.""" + """The name of the directory storing run-related files.""" log_directory = "log" - """The name of the directory where log files are stored.""" + """The name of the directory storing log-related files.""" def __init__(self, root: pathlib.Path) -> None: """Initialize a Generator object - The Generator class constructs a Job's directory structure, including: + The Generator class is responsible for constructing a Job's directory, performing + the following tasks: - - The run and log directories - - Output and error files - - The "smartsim_params.txt" settings file + - Creating the run and log directories + - Generating the output and error files + - Building the parameter settings file + - Managing symlinking, copying, and configuration of attached files - Additionally, it manages symlinking, copying, and configuring files associated - with a Job's entity. - - :param root: Job base path + :param root: The base path for job-related files and directories """ self.root = root - """The root path under which to generate files""" + """The root directory under which all generated files and directories will be placed.""" def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's base directory. The path is created by combining the @@ -98,8 +103,8 @@ def _build_job_base_path(self, job: Job, job_index: int) -> pathlib.Path: def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's run directory. The path is formed by combining - the base directory with the `run` class-level variable, where run specifies - the name of the job's run folder. + the base directory with the `run_directory` class-level constant, which specifies + the name of the Job's run folder. :param job: Job object :param job_index: Job index @@ -110,8 +115,8 @@ def _build_job_run_path(self, job: Job, job_index: int) -> pathlib.Path: def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path: """Build and return a Job's log directory. The path is formed by combining - the base directory with the `log` class-level variable, where log specifies - the name of the job's log folder. + the base directory with the `log_directory` class-level constant, which specifies + the name of the Job's log folder. :param job: Job object :param job_index: Job index @@ -122,7 +127,7 @@ def _build_job_log_path(self, job: Job, job_index: int) -> pathlib.Path: @staticmethod def _build_log_file_path(log_path: pathlib.Path) -> pathlib.Path: - """Build and return an entities file summarizing the parameters + """Build and return a parameters file summarizing the parameters used for the generation of the entity. :param log_path: Path to log directory @@ -155,7 +160,7 @@ def _build_err_file_path(log_path: pathlib.Path, job_name: str) -> pathlib.Path: return err_file_path def generate_job(self, job: Job, job_index: int) -> Job_Path: - """Build and return the Job's run directory, error file and out file. + """Build and return the Job's run directory, output file, and error file. This method creates the Job's run and log directories, generates the `smartsim_params.txt` file to log parameters used for the Job, and sets @@ -174,7 +179,7 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path: out_file = self._build_out_file_path(log_path, job.entity.name) err_file = self._build_err_file_path(log_path, job.entity.name) - cmd_list = self._build_commands(job, job_path, log_path) + cmd_list = self._build_commands(job.entity, job_path, log_path) self._execute_commands(cmd_list) @@ -188,7 +193,10 @@ def generate_job(self, job: Job, job_index: int) -> Job_Path: @classmethod def _build_commands( - cls, job: Job, job_path: pathlib.Path, log_path: pathlib.Path + cls, + entity: entity.SmartSimEntity, + job_path: pathlib.Path, + log_path: pathlib.Path, ) -> CommandList: """Build file operation commands for a Job's entity. @@ -199,33 +207,55 @@ def _build_commands( :param job: Job object :param job_path: The file path for the Job run folder + :param log_path: The file path for the Job log folder :return: A CommandList containing the file operation commands """ + context = GenerationContext(job_path) cmd_list = CommandList() - cmd_list.commands.append(cls._mkdir_file(job_path)) - cmd_list.commands.append(cls._mkdir_file(log_path)) - entity = job.entity + + cls._append_mkdir_commands(cmd_list, job_path, log_path) + if isinstance(entity, _GenerableProtocol): - helpers: t.List[ - t.Callable[ - [t.Union[EntityFiles, None], pathlib.Path], - t.Union[CommandList, None], - ] - ] = [ - cls._copy_files, - cls._symlink_files, - lambda files, path: cls._write_tagged_files( - files, entity.file_parameters, path - ), - ] - - for method in helpers: - return_cmd_list = method(entity.files, job_path) - if return_cmd_list: - cmd_list.commands.extend(return_cmd_list.commands) + cls._append_file_operations(cmd_list, entity, context) return cmd_list + @classmethod + def _append_mkdir_commands( + cls, cmd_list: CommandList, job_path: pathlib.Path, log_path: pathlib.Path + ) -> None: + """Append file operation Commands (mkdir) for a Job's run and log directory. + + :param cmd_list: A CommandList object containing the commands to be executed + :param job_path: The file path for the Job run folder + :param log_path: The file path for the Job log folder + """ + cmd_list.append(cls._mkdir_file(job_path)) + cmd_list.append(cls._mkdir_file(log_path)) + + @classmethod + def _append_file_operations( + cls, + cmd_list: CommandList, + entity: _GenerableProtocol, + context: GenerationContext, + ) -> None: + """Append file operation Commands (copy, symlink, configure) for all + files attached to the entity. + + :param cmd_list: A CommandList object containing the commands to be executed + :param entity: The Job's attached entity + :param context: A GenerationContext object that holds the Job's run directory + """ + copy_ret = cls._copy_files(entity.files.copy_operations, context) + cmd_list.extend(copy_ret) + + symlink_ret = cls._symlink_files(entity.files.symlink_operations, context) + cmd_list.extend(symlink_ret) + + configure_ret = cls._configure_files(entity.files.configure_operations, context) + cmd_list.extend(configure_ret) + @classmethod def _execute_commands(cls, cmd_list: CommandList) -> None: """Execute a list of commands using subprocess. @@ -240,119 +270,51 @@ def _execute_commands(cls, cmd_list: CommandList) -> None: @staticmethod def _mkdir_file(file_path: pathlib.Path) -> Command: + """Build a Command to create the directory along with any + necessary parent directories. + + :param file_path: The directory path to be created + :return: A Command object to execute the directory creation + """ cmd = Command(["mkdir", "-p", str(file_path)]) return cmd @staticmethod def _copy_files( - files: t.Union[EntityFiles, None], dest: pathlib.Path - ) -> t.Optional[CommandList]: - """Build command to copy files/directories from specified paths to a destination directory. - - This method creates commands to copy files/directories from the source paths provided in the - `files` parameter to the specified destination directory. If the source is a directory, - it copies the directory while allowing existing directories to remain intact. + files: list[CopyOperation], context: GenerationContext + ) -> CommandList: + """Build commands to copy files/directories from specified source paths + to an optional destination in the run directory. - :param files: An EntityFiles object containing the paths to copy, or None. - :param dest: The destination path to the Job's run directory. - :return: A CommandList containing the copy commands, or None if no files are provided. + :param files: A list of CopyOperation objects + :param context: A GenerationContext object that holds the Job's run directory + :return: A CommandList containing the copy commands """ - if files is None: - return None - cmd_list = CommandList() - for src in files.copy: - cmd = Command( - [ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "copy", - src, - ] - ) - destination = str(dest) - if os.path.isdir(src): - base_source_name = os.path.basename(src) - destination = os.path.join(dest, base_source_name) - cmd.append(str(destination)) - cmd.append("--dirs_exist_ok") - else: - cmd.append(str(dest)) - cmd_list.commands.append(cmd) - return cmd_list + return CommandList([file.format(context) for file in files]) @staticmethod def _symlink_files( - files: t.Union[EntityFiles, None], dest: pathlib.Path - ) -> t.Optional[CommandList]: - """Build command to symlink files/directories from specified paths to a destination directory. - - This method creates commands to symlink files/directories from the source paths provided in the - `files` parameter to the specified destination directory. If the source is a directory, - it copies the directory while allowing existing directories to remain intact. + files: list[SymlinkOperation], context: GenerationContext + ) -> CommandList: + """Build commands to symlink files/directories from specified source paths + to an optional destination in the run directory. - :param files: An EntityFiles object containing the paths to symlink, or None. - :param dest: The destination path to the Job's run directory. - :return: A CommandList containing the symlink commands, or None if no files are provided. + :param files: A list of SymlinkOperation objects + :param context: A GenerationContext object that holds the Job's run directory + :return: A CommandList containing the symlink commands """ - if files is None: - return None - cmd_list = CommandList() - for src in files.link: - # Normalize the path to remove trailing slashes - normalized_path = os.path.normpath(src) - # Get the parent directory (last folder) - parent_dir = os.path.basename(normalized_path) - new_dest = os.path.join(str(dest), parent_dir) - cmd = Command( - [ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "symlink", - src, - new_dest, - ] - ) - cmd_list.append(cmd) - return cmd_list + return CommandList([file.format(context) for file in files]) @staticmethod - def _write_tagged_files( - files: t.Union[EntityFiles, None], - params: t.Mapping[str, str], - dest: pathlib.Path, - ) -> t.Optional[CommandList]: - """Build command to configure files/directories from specified paths to a destination directory. - - This method processes tagged files by reading their configurations, - serializing the provided parameters, and generating commands to - write these configurations to the destination directory. - - :param files: An EntityFiles object containing the paths to configure, or None. - :param params: A dictionary of params - :param dest: The destination path to the Job's run directory. - :return: A CommandList containing the configuration commands, or None if no files are provided. + def _configure_files( + files: list[ConfigureOperation], + context: GenerationContext, + ) -> CommandList: + """Build commands to configure files/directories from specified source paths + to an optional destination in the run directory. + + :param files: A list of ConfigurationOperation objects + :param context: A GenerationContext object that holds the Job's run directory + :return: A CommandList containing the configuration commands """ - if files is None: - return None - cmd_list = CommandList() - if files.tagged: - tag_delimiter = ";" - pickled_dict = pickle.dumps(params) - encoded_dict = base64.b64encode(pickled_dict).decode("ascii") - for path in files.tagged: - cmd = Command( - [ - sys.executable, - "-m", - "smartsim._core.entrypoints.file_operations", - "configure", - path, - str(dest), - tag_delimiter, - encoded_dict, - ] - ) - cmd_list.commands.append(cmd) - return cmd_list + return CommandList([file.format(context) for file in files]) diff --git a/smartsim/_core/generation/operations/operations.py b/smartsim/_core/generation/operations/operations.py new file mode 100644 index 0000000000..48ccc6c7b2 --- /dev/null +++ b/smartsim/_core/generation/operations/operations.py @@ -0,0 +1,280 @@ +import base64 +import os +import pathlib +import pickle +import sys +import typing as t +from dataclasses import dataclass, field + +from ...commands import Command +from .utils.helpers import check_src_and_dest_path + +# pylint: disable-next=invalid-name +entry_point_path = "smartsim._core.entrypoints.file_operations" +"""Path to file operations module""" + +# pylint: disable-next=invalid-name +copy_cmd = "copy" +"""Copy file operation command""" +# pylint: disable-next=invalid-name +symlink_cmd = "symlink" +"""Symlink file operation command""" +# pylint: disable-next=invalid-name +configure_cmd = "configure" +"""Configure file operation command""" + +# pylint: disable-next=invalid-name +default_tag = ";" +"""Default configure tag""" + + +def _create_dest_path(job_run_path: pathlib.Path, dest: pathlib.Path) -> str: + """Combine the job run path and destination path. Return as a string for + entry point consumption. + + :param job_run_path: Job run path + :param dest: Destination path + :return: Combined path + """ + return str(job_run_path / dest) + + +def _check_run_path(run_path: pathlib.Path) -> None: + """Validate that the provided run path is of type pathlib.Path + + :param run_path: The run path to be checked + :raises TypeError: If either run path is not an instance of pathlib.Path + :raises ValueError: If the run path is not a directory + """ + if not isinstance(run_path, pathlib.Path): + raise TypeError( + f"The Job's run path must be of type pathlib.Path, not {type(run_path).__name__}" + ) + if not run_path.is_absolute(): + raise ValueError(f"The Job's run path must be absolute.") + + +class GenerationContext: + """Context for file system generation operations.""" + + def __init__(self, job_run_path: pathlib.Path): + """Initialize a GenerationContext object + + :param job_run_path: Job's run path + """ + _check_run_path(job_run_path) + self.job_run_path = job_run_path + """The Job run path""" + + +class GenerationProtocol(t.Protocol): + """Protocol for Generation Operations.""" + + def format(self, context: GenerationContext) -> Command: + """Return a formatted Command.""" + + +class CopyOperation(GenerationProtocol): + """Copy Operation""" + + def __init__( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Initialize a CopyOperation object + + :param src: Path to source + :param dest: Path to destination + """ + check_src_and_dest_path(src, dest) + self.src = src + """Path to source""" + self.dest = dest or pathlib.Path(src.name) + """Path to destination""" + + def format(self, context: GenerationContext) -> Command: + """Create Command to invoke copy file system entry point + + :param context: Context for copy operation + :return: Copy Command + """ + final_dest = _create_dest_path(context.job_run_path, self.dest) + return Command( + [ + sys.executable, + "-m", + entry_point_path, + copy_cmd, + str(self.src), + final_dest, + "--dirs_exist_ok", + ] + ) + + +class SymlinkOperation(GenerationProtocol): + """Symlink Operation""" + + def __init__( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Initialize a SymlinkOperation object + + :param src: Path to source + :param dest: Path to destination + """ + check_src_and_dest_path(src, dest) + self.src = src + """Path to source""" + self.dest = dest or pathlib.Path(src.name) + """Path to destination""" + + def format(self, context: GenerationContext) -> Command: + """Create Command to invoke symlink file system entry point + + :param context: Context for symlink operation + :return: Symlink Command + """ + normalized_path = os.path.normpath(self.src) + parent_dir = os.path.dirname(normalized_path) + final_dest = _create_dest_path(context.job_run_path, self.dest) + new_dest = os.path.join(final_dest, parent_dir) + return Command( + [ + sys.executable, + "-m", + entry_point_path, + symlink_cmd, + str(self.src), + new_dest, + ] + ) + + +class ConfigureOperation(GenerationProtocol): + """Configure Operation""" + + def __init__( + self, + src: pathlib.Path, + file_parameters: t.Mapping[str, str], + dest: t.Optional[pathlib.Path] = None, + tag: t.Optional[str] = None, + ) -> None: + """Initialize a ConfigureOperation + + :param src: Path to source + :param file_parameters: File parameters to find and replace + :param dest: Path to destination + :param tag: Tag to use for find and replacement + """ + check_src_and_dest_path(src, dest) + self.src = src + """Path to source""" + self.dest = dest or pathlib.Path(src.name) + """Path to destination""" + pickled_dict = pickle.dumps(file_parameters) + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + self.file_parameters = encoded_dict + """File parameters to find and replace""" + self.tag = tag if tag else default_tag + """Tag to use for find and replacement""" + + def format(self, context: GenerationContext) -> Command: + """Create Command to invoke configure file system entry point + + :param context: Context for configure operation + :return: Configure Command + """ + final_dest = _create_dest_path(context.job_run_path, self.dest) + return Command( + [ + sys.executable, + "-m", + entry_point_path, + configure_cmd, + str(self.src), + final_dest, + self.tag, + self.file_parameters, + ] + ) + + +GenerationProtocolT = t.TypeVar("GenerationProtocolT", bound=GenerationProtocol) + + +@dataclass +class FileSysOperationSet: + """Dataclass to represent a set of file system operation objects""" + + operations: list[GenerationProtocol] = field(default_factory=list) + """Set of file system objects that match the GenerationProtocol""" + + def add_copy( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Add a copy operation to the operations list + + :param src: Path to source + :param dest: Path to destination + """ + self.operations.append(CopyOperation(src, dest)) + + def add_symlink( + self, src: pathlib.Path, dest: t.Optional[pathlib.Path] = None + ) -> None: + """Add a symlink operation to the operations list + + :param src: Path to source + :param dest: Path to destination + """ + self.operations.append(SymlinkOperation(src, dest)) + + def add_configuration( + self, + src: pathlib.Path, + file_parameters: t.Mapping[str, str], + dest: t.Optional[pathlib.Path] = None, + tag: t.Optional[str] = None, + ) -> None: + """Add a configure operation to the operations list + + :param src: Path to source + :param file_parameters: File parameters to find and replace + :param dest: Path to destination + :param tag: Tag to use for find and replacement + """ + self.operations.append(ConfigureOperation(src, file_parameters, dest, tag)) + + @property + def copy_operations(self) -> list[CopyOperation]: + """Property to get the list of copy files. + + :return: List of CopyOperation objects + """ + return self._filter(CopyOperation) + + @property + def symlink_operations(self) -> list[SymlinkOperation]: + """Property to get the list of symlink files. + + :return: List of SymlinkOperation objects + """ + return self._filter(SymlinkOperation) + + @property + def configure_operations(self) -> list[ConfigureOperation]: + """Property to get the list of configure files. + + :return: List of ConfigureOperation objects + """ + return self._filter(ConfigureOperation) + + def _filter(self, type_: type[GenerationProtocolT]) -> list[GenerationProtocolT]: + """Filters the operations list to include only instances of the + specified type. + + :param type: The type of operations to filter + :return: A list of operations that are instances of the specified type + """ + return [x for x in self.operations if isinstance(x, type_)] diff --git a/smartsim/_core/generation/operations/utils/helpers.py b/smartsim/_core/generation/operations/utils/helpers.py new file mode 100644 index 0000000000..9d99b0e8bf --- /dev/null +++ b/smartsim/_core/generation/operations/utils/helpers.py @@ -0,0 +1,27 @@ +import pathlib +import typing as t + + +def check_src_and_dest_path( + src: pathlib.Path, dest: t.Union[pathlib.Path, None] +) -> None: + """Validate that the provided source and destination paths are + of type pathlib.Path. Additionally, validate that destination is a + relative Path and source is a absolute Path. + + :param src: The source path to check + :param dest: The destination path to check + :raises TypeError: If either src or dest is not of type pathlib.Path + :raises ValueError: If source is not an absolute Path or if destination is not + a relative Path + """ + if not isinstance(src, pathlib.Path): + raise TypeError(f"src must be of type pathlib.Path, not {type(src).__name__}") + if dest is not None and not isinstance(dest, pathlib.Path): + raise TypeError( + f"dest must be of type pathlib.Path or None, not {type(dest).__name__}" + ) + if dest is not None and dest.is_absolute(): + raise ValueError(f"dest must be a relative Path") + if not src.is_absolute(): + raise ValueError(f"src must be an absolute Path") diff --git a/smartsim/builders/ensemble.py b/smartsim/builders/ensemble.py index d8a16880be..d87ada15aa 100644 --- a/smartsim/builders/ensemble.py +++ b/smartsim/builders/ensemble.py @@ -387,7 +387,6 @@ def _create_applications(self) -> tuple[Application, ...]: name=f"{self.name}-{i}", exe=self.exe, exe_args=self.exe_args, - files=self.files, file_parameters=permutation.params, ) for i, permutation in enumerate(permutations_) diff --git a/smartsim/entity/application.py b/smartsim/entity/application.py index 402f0aa30a..501279c85f 100644 --- a/smartsim/entity/application.py +++ b/smartsim/entity/application.py @@ -32,10 +32,10 @@ import typing as t from os import path as osp +from .._core.generation.operations.operations import FileSysOperationSet from .._core.utils.helpers import expand_exe_path from ..log import get_logger from .entity import SmartSimEntity -from .files import EntityFiles logger = get_logger(__name__) @@ -59,8 +59,9 @@ def __init__( name: str, exe: str, exe_args: t.Optional[t.Union[str, t.Sequence[str]]] = None, - files: t.Optional[EntityFiles] = None, - file_parameters: t.Mapping[str, str] | None = None, + file_parameters: ( + t.Mapping[str, str] | None + ) = None, # TODO remove when Ensemble is addressed ) -> None: """Initialize an ``Application`` @@ -77,10 +78,6 @@ def __init__( :param name: name of the application :param exe: executable to run :param exe_args: executable arguments - :param files: files to be copied, symlinked, and/or configured prior to - execution - :param file_parameters: parameters and values to be used when configuring - files """ super().__init__(name) """The name of the application""" @@ -88,12 +85,13 @@ def __init__( """The executable to run""" self._exe_args = self._build_exe_args(exe_args) or [] """The executable arguments""" - self._files = copy.deepcopy(files) if files else EntityFiles() - """Files to be copied, symlinked, and/or configured prior to execution""" + self.files = FileSysOperationSet([]) + """Attach files""" self._file_parameters = ( copy.deepcopy(file_parameters) if file_parameters else {} ) - """Parameters and values to be used when configuring files""" + """TODO MOCK until Ensemble is implemented""" + """Files to be copied, symlinked, and/or configured prior to execution""" self._incoming_entities: t.List[SmartSimEntity] = [] """Entities for which the prefix will have to be known by other entities""" self._key_prefixing_enabled = False @@ -147,30 +145,6 @@ def add_exe_args(self, args: t.Union[str, t.List[str], None]) -> None: args = self._build_exe_args(args) self._exe_args.extend(args) - @property - def files(self) -> t.Union[EntityFiles, None]: - """Return attached EntityFiles object. - - :return: the EntityFiles object of files to be copied, symlinked, - and/or configured prior to execution - """ - return self._files - - @files.setter - def files(self, value: EntityFiles) -> None: - """Set the EntityFiles object. - - :param value: the EntityFiles object of files to be copied, symlinked, - and/or configured prior to execution - :raises TypeError: files argument was not of type int - - """ - - if not isinstance(value, EntityFiles): - raise TypeError("files argument was not of type EntityFiles") - - self._files = copy.deepcopy(value) - @property def file_parameters(self) -> t.Mapping[str, str]: """Return file parameters. @@ -249,60 +223,6 @@ def as_executable_sequence(self) -> t.Sequence[str]: """ return [self.exe, *self.exe_args] - def attach_generator_files( - self, - to_copy: t.Optional[t.List[str]] = None, - to_symlink: t.Optional[t.List[str]] = None, - to_configure: t.Optional[t.List[str]] = None, - ) -> None: - """Attach files to an entity for generation - - Attach files needed for the entity that, upon generation, - will be located in the path of the entity. Invoking this method - after files have already been attached will overwrite - the previous list of entity files. - - During generation, files "to_copy" are copied into - the path of the entity, and files "to_symlink" are - symlinked into the path of the entity. - - Files "to_configure" are text based application input files where - parameters for the application are set. Note that only applications - support the "to_configure" field. These files must have - fields tagged that correspond to the values the user - would like to change. The tag is settable but defaults - to a semicolon e.g. THERMO = ;10; - - :param to_copy: files to copy - :param to_symlink: files to symlink - :param to_configure: input files with tagged parameters - :raises ValueError: if the generator file already exists - """ - to_copy = to_copy or [] - to_symlink = to_symlink or [] - to_configure = to_configure or [] - - # Check that no file collides with the parameter file written - # by Generator. We check the basename, even though it is more - # restrictive than what we need (but it avoids relative path issues) - for strategy in [to_copy, to_symlink, to_configure]: - if strategy is not None and any( - osp.basename(filename) == "smartsim_params.txt" for filename in strategy - ): - raise ValueError( - "`smartsim_params.txt` is a file automatically " - + "generated by SmartSim and cannot be ovewritten." - ) - self.files = EntityFiles(to_configure, to_copy, to_symlink) - - @property - def attached_files_table(self) -> str: - """Return a list of attached files as a plain text table - - :return: String version of table - """ - return str(self.files) - @staticmethod def _build_exe_args(exe_args: t.Union[str, t.Sequence[str], None]) -> t.List[str]: """Check and convert exe_args input to a desired collection format @@ -327,10 +247,6 @@ def _build_exe_args(exe_args: t.Union[str, t.Sequence[str], None]) -> t.List[str return list(exe_args) - def print_attached_files(self) -> None: - """Print a table of the attached files on std out""" - print(self.attached_files_table) - def __str__(self) -> str: # pragma: no cover exe_args_str = "\n".join(self.exe_args) entities_str = "\n".join(str(entity) for entity in self.incoming_entities) @@ -341,8 +257,6 @@ def __str__(self) -> str: # pragma: no cover {self.exe} Executable Arguments: {exe_args_str} - Entity Files: {self.files} - File Parameters: {self.file_parameters} Incoming Entities: {entities_str} Key Prefixing Enabled: {self.key_prefixing_enabled} diff --git a/smartsim/entity/files.py b/smartsim/entity/files.py index 08143fbfc2..42586f153e 100644 --- a/smartsim/entity/files.py +++ b/smartsim/entity/files.py @@ -29,6 +29,7 @@ from tabulate import tabulate +# TODO remove when Ensemble is addressed class EntityFiles: """EntityFiles are the files a user wishes to have available to applications and nodes within SmartSim. Each entity has a method diff --git a/tests/temp_tests/test_core/test_commands/test_command.py b/tests/temp_tests/test_core/test_commands/test_command.py index 2d1ddfbe84..f3d6f6a2a3 100644 --- a/tests/temp_tests/test_core/test_commands/test_command.py +++ b/tests/temp_tests/test_core/test_commands/test_command.py @@ -36,10 +36,16 @@ def test_command_init(): assert cmd.command == ["salloc", "-N", "1"] -def test_command_getitem_int(): +def test_command_invalid_init(): cmd = Command(command=["salloc", "-N", "1"]) - get_value = cmd[0] - assert get_value == "salloc" + assert cmd.command == ["salloc", "-N", "1"] + + +def test_command_getitem_int(): + with pytest.raises(TypeError): + _ = Command(command=[1]) + with pytest.raises(TypeError): + _ = Command(command=[]) def test_command_getitem_slice(): @@ -63,9 +69,9 @@ def test_command_setitem_slice(): def test_command_setitem_fail(): cmd = Command(command=["salloc", "-N", "1"]) - with pytest.raises(ValueError): + with pytest.raises(TypeError): cmd[0] = 1 - with pytest.raises(ValueError): + with pytest.raises(TypeError): cmd[0:2] = [1, "-n"] diff --git a/tests/temp_tests/test_core/test_commands/test_commandList.py b/tests/temp_tests/test_core/test_commands/test_commandList.py index c6bc8d8347..37acefd8d3 100644 --- a/tests/temp_tests/test_core/test_commands/test_commandList.py +++ b/tests/temp_tests/test_core/test_commands/test_commandList.py @@ -70,19 +70,16 @@ def test_command_setitem_slice(): def test_command_setitem_fail(): cmd_list = CommandList(commands=[srun_cmd, srun_cmd]) - with pytest.raises(ValueError): + with pytest.raises(TypeError): cmd_list[0] = "fail" - with pytest.raises(ValueError): + with pytest.raises(TypeError): cmd_list[0:1] = "fail" - with pytest.raises(ValueError): + with pytest.raises(TypeError): cmd_list[0:1] = "fail" - cmd_1 = Command(command=["salloc", "-N", 1]) - cmd_2 = Command(command=["salloc", "-N", "1"]) - cmd_3 = Command(command=1) - with pytest.raises(ValueError): - cmd_list[0:1] = [cmd_1, cmd_2] - with pytest.raises(ValueError): - cmd_list[0:1] = [cmd_3, cmd_2] + with pytest.raises(TypeError): + _ = Command(command=["salloc", "-N", 1]) + with pytest.raises(TypeError): + cmd_list[0:1] = [Command(command=["salloc", "-N", "1"]), Command(command=1)] def test_command_delitem(): diff --git a/tests/test_application.py b/tests/test_application.py index d329321504..54a02c5b4d 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -30,7 +30,6 @@ import pytest from smartsim.entity.application import Application -from smartsim.entity.files import EntityFiles from smartsim.settings.launch_settings import LaunchSettings pytestmark = pytest.mark.group_a @@ -62,14 +61,6 @@ def test_application_exe_args_property(): assert exe_args is a.exe_args -def test_application_files_property(get_gen_configure_dir): - tagged_files = sorted(glob(get_gen_configure_dir + "/*")) - files = EntityFiles(tagged=tagged_files) - a = Application("test_name", exe="echo", exe_args=["spam", "eggs"], files=files) - files = a.files - assert files is a.files - - def test_application_file_parameters_property(): file_parameters = {"h": [5, 6, 7, 8]} a = Application( @@ -120,24 +111,6 @@ def test_type_exe_args(): application.exe_args = [1, 2, 3] -def test_type_files_property(): - application = Application( - "test_name", - exe="echo", - ) - with pytest.raises(TypeError): - application.files = "/path/to/file" - - -def test_type_file_parameters_property(): - application = Application( - "test_name", - exe="echo", - ) - with pytest.raises(TypeError): - application.file_parameters = {1: 2} - - def test_type_incoming_entities(): application = Application( "test_name", @@ -171,16 +144,6 @@ def test_application_type_exe_args(): application.exe_args = [1, 2, 3] -def test_application_type_files(): - application = Application( - "test_name", - exe="echo", - exe_args=["spam", "eggs"], - ) - with pytest.raises(TypeError, match="files argument was not of type EntityFiles"): - application.files = 2 - - @pytest.mark.parametrize( "file_params", ( diff --git a/tests/test_configs/generator_files/to_copy_dir/mock.txt b/tests/test_configs/generator_files/to_copy_dir/mock_1.txt similarity index 100% rename from tests/test_configs/generator_files/to_copy_dir/mock.txt rename to tests/test_configs/generator_files/to_copy_dir/mock_1.txt diff --git a/tests/test_configs/generator_files/to_copy_dir/mock_2.txt b/tests/test_configs/generator_files/to_copy_dir/mock_2.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_configs/generator_files/to_copy_dir/mock_3.txt b/tests/test_configs/generator_files/to_copy_dir/mock_3.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_configs/generator_files/to_symlink_dir/mock_1.txt b/tests/test_configs/generator_files/to_symlink_dir/mock_1.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_configs/generator_files/to_symlink_dir/mock2.txt b/tests/test_configs/generator_files/to_symlink_dir/mock_2.txt similarity index 100% rename from tests/test_configs/generator_files/to_symlink_dir/mock2.txt rename to tests/test_configs/generator_files/to_symlink_dir/mock_2.txt diff --git a/tests/test_configs/generator_files/to_symlink_dir/mock_3.txt b/tests/test_configs/generator_files/to_symlink_dir/mock_3.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_generator.py b/tests/test_generator.py index 4c25ccd05f..3915526a8b 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -25,25 +25,24 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import itertools -import os import pathlib import unittest.mock from glob import glob -from os import listdir from os import path as osp import pytest -from smartsim import Experiment from smartsim._core.commands import Command, CommandList from smartsim._core.generation.generator import Generator -from smartsim.builders import Ensemble -from smartsim.entity import entity -from smartsim.entity.files import EntityFiles +from smartsim._core.generation.operations.operations import ( + ConfigureOperation, + CopyOperation, + FileSysOperationSet, + GenerationContext, + SymlinkOperation, +) +from smartsim.entity import SmartSimEntity from smartsim.launchable import Job -from smartsim.settings import LaunchSettings - -# TODO Add JobGroup tests when JobGroup becomes a Launchable pytestmark = pytest.mark.group_a @@ -57,47 +56,30 @@ def random_id(): return next(_ID_GENERATOR) -@pytest.fixture -def get_gen_copy_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) - - -@pytest.fixture -def get_gen_symlink_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) - - -@pytest.fixture -def get_gen_configure_dir(fileutils): - yield fileutils.get_test_conf_path(osp.join("generator_files", "tag_dir_template")) - - @pytest.fixture def generator_instance(test_dir: str) -> Generator: - """Fixture to create an instance of Generator.""" - root = pathlib.Path(test_dir, "temp_id") - os.mkdir(root) - yield Generator(root=root) + """Instance of Generator""" + # os.mkdir(root) + yield Generator(root=pathlib.Path(test_dir)) -def get_gen_file(fileutils, filename: str): - return fileutils.get_test_conf_path(osp.join("generator_files", filename)) +@pytest.fixture +def mock_index(): + """Fixture to create a mock destination path.""" + return 1 -class EchoHelloWorldEntity(entity.SmartSimEntity): +class EchoHelloWorldEntity(SmartSimEntity): """A simple smartsim entity that meets the `ExecutableProtocol` protocol""" def __init__(self): self.name = "entity_name" - self.files = None + self.files = FileSysOperationSet([]) self.file_parameters = None def as_executable_sequence(self): return ("echo", "Hello", "World!") - def files(): - return ["file_path"] - @pytest.fixture def mock_job() -> unittest.mock.MagicMock: @@ -120,14 +102,13 @@ def mock_job() -> unittest.mock.MagicMock: def test_init_generator(generator_instance: Generator, test_dir: str): """Test Generator init""" - assert generator_instance.root == pathlib.Path(test_dir) / "temp_id" + assert generator_instance.root == pathlib.Path(test_dir) def test_build_job_base_path( - generator_instance: Generator, mock_job: unittest.mock.MagicMock + generator_instance: Generator, mock_job: unittest.mock.MagicMock, mock_index ): """Test Generator._build_job_base_path returns correct path""" - mock_index = 1 root_path = generator_instance._build_job_base_path(mock_job, mock_index) expected_path = ( generator_instance.root @@ -142,16 +123,16 @@ def test_build_job_run_path( mock_job: unittest.mock.MagicMock, generator_instance: Generator, monkeypatch: pytest.MonkeyPatch, + mock_index, ): """Test Generator._build_job_run_path returns correct path""" - mock_index = 1 monkeypatch.setattr( Generator, "_build_job_base_path", lambda self, job, job_index: pathlib.Path(test_dir), ) run_path = generator_instance._build_job_run_path(mock_job, mock_index) - expected_run_path = pathlib.Path(test_dir) / "run" + expected_run_path = pathlib.Path(test_dir) / generator_instance.run_directory assert run_path == expected_run_path @@ -160,16 +141,16 @@ def test_build_job_log_path( mock_job: unittest.mock.MagicMock, generator_instance: Generator, monkeypatch: pytest.MonkeyPatch, + mock_index, ): """Test Generator._build_job_log_path returns correct path""" - mock_index = 1 monkeypatch.setattr( Generator, "_build_job_base_path", lambda self, job, job_index: pathlib.Path(test_dir), ) log_path = generator_instance._build_job_log_path(mock_job, mock_index) - expected_log_path = pathlib.Path(test_dir) / "log" + expected_log_path = pathlib.Path(test_dir) / generator_instance.log_directory assert log_path == expected_log_path @@ -200,42 +181,15 @@ def test_build_err_file_path( def test_generate_job( - mock_job: unittest.mock.MagicMock, - generator_instance: Generator, + mock_job: unittest.mock.MagicMock, generator_instance: Generator, mock_index: int ): """Test Generator.generate_job returns correct paths""" - mock_index = 1 job_paths = generator_instance.generate_job(mock_job, mock_index) assert job_paths.run_path.name == Generator.run_directory assert job_paths.out_path.name == f"{mock_job.entity.name}.out" assert job_paths.err_path.name == f"{mock_job.entity.name}.err" -def test_build_commands( - mock_job: unittest.mock.MagicMock, generator_instance: Generator, test_dir: str -): - """Test Generator._build_commands calls correct helper functions""" - with ( - unittest.mock.patch( - "smartsim._core.generation.Generator._copy_files" - ) as mock_copy_files, - unittest.mock.patch( - "smartsim._core.generation.Generator._symlink_files" - ) as mock_symlink_files, - unittest.mock.patch( - "smartsim._core.generation.Generator._write_tagged_files" - ) as mock_write_tagged_files, - ): - generator_instance._build_commands( - mock_job, - pathlib.Path(test_dir) / generator_instance.run_directory, - pathlib.Path(test_dir) / generator_instance.log_directory, - ) - mock_copy_files.assert_called_once() - mock_symlink_files.assert_called_once() - mock_write_tagged_files.assert_called_once() - - def test_execute_commands(generator_instance: Generator): """Test Generator._execute_commands subprocess.run""" with ( @@ -255,55 +209,75 @@ def test_mkdir_file(generator_instance: Generator, test_dir: str): assert cmd.command == ["mkdir", "-p", test_dir] -def test_copy_file(generator_instance: Generator, fileutils): - """Test Generator._copy_files helper function with file""" - script = fileutils.get_test_conf_path("sleep.py") - files = EntityFiles(copy=script) - cmd_list = generator_instance._copy_files(files, generator_instance.root) - assert isinstance(cmd_list, CommandList) - assert len(cmd_list) == 1 - assert str(generator_instance.root) and script in cmd_list.commands[0].command - - -def test_copy_directory(get_gen_copy_dir, generator_instance: Generator): - """Test Generator._copy_files helper function with directory""" - files = EntityFiles(copy=get_gen_copy_dir) - cmd_list = generator_instance._copy_files(files, generator_instance.root) - assert isinstance(cmd_list, CommandList) - assert len(cmd_list) == 1 - assert ( - str(generator_instance.root) - and get_gen_copy_dir in cmd_list.commands[0].command - ) - - -def test_symlink_file(get_gen_symlink_dir, generator_instance: Generator): - """Test Generator._symlink_files helper function with file list""" - symlink_files = sorted(glob(get_gen_symlink_dir + "/*")) - files = EntityFiles(symlink=symlink_files) - cmd_list = generator_instance._symlink_files(files, generator_instance.root) +@pytest.mark.parametrize( + "dest", + ( + pytest.param(None, id="dest as None"), + pytest.param( + pathlib.Path("absolute/path"), + id="dest as valid path", + ), + ), +) +def test_copy_files_valid_dest( + dest, source, generator_instance: Generator, test_dir: str +): + to_copy = [CopyOperation(src=file, dest=dest) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + cmd_list = generator_instance._copy_files(files=to_copy, context=gen) assert isinstance(cmd_list, CommandList) - for file, cmd in zip(symlink_files, cmd_list): - assert file in cmd.command - - -def test_symlink_directory(generator_instance: Generator, get_gen_symlink_dir): - """Test Generator._symlink_files helper function with directory""" - files = EntityFiles(symlink=get_gen_symlink_dir) - cmd_list = generator_instance._symlink_files(files, generator_instance.root) - symlinked_folder = generator_instance.root / os.path.basename(get_gen_symlink_dir) + # Extract file paths from commands + cmd_src_paths = set() + for cmd in cmd_list.commands: + src_index = cmd.command.index("copy") + 1 + cmd_src_paths.add(cmd.command[src_index]) + # Assert all file paths are in the command list + file_paths = {str(file) for file in source} + assert file_paths == cmd_src_paths, "Not all file paths are in the command list" + + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(None, id="dest as None"), + pytest.param( + pathlib.Path("absolute/path"), + id="dest as valid path", + ), + ), +) +def test_symlink_files_valid_dest( + dest, source, generator_instance: Generator, test_dir: str +): + to_symlink = [SymlinkOperation(src=file, dest=dest) for file in source] + gen = GenerationContext(pathlib.Path(test_dir)) + cmd_list = generator_instance._symlink_files(files=to_symlink, context=gen) assert isinstance(cmd_list, CommandList) - assert str(symlinked_folder) in cmd_list.commands[0].command - - -def test_write_tagged_file(fileutils, generator_instance: Generator): - """Test Generator._write_tagged_files helper function with file list""" - conf_path = fileutils.get_test_conf_path( - osp.join("generator_files", "easy", "marked/") - ) - tagged_files = sorted(glob(conf_path + "/*")) - files = EntityFiles(tagged=tagged_files) - param_set = { + # Extract file paths from commands + cmd_src_paths = set() + for cmd in cmd_list.commands: + print(cmd) + src_index = cmd.command.index("symlink") + 1 + cmd_src_paths.add(cmd.command[src_index]) + # Assert all file paths are in the command list + file_paths = {str(file) for file in source} + assert file_paths == cmd_src_paths, "Not all file paths are in the command list" + + +@pytest.mark.parametrize( + "dest", + ( + pytest.param(None, id="dest as None"), + pytest.param( + pathlib.Path("absolute/path"), + id="dest as valid path", + ), + ), +) +def test_configure_files_valid_dest( + dest, source, generator_instance: Generator, test_dir: str +): + file_param = { "5": 10, "FIRST": "SECOND", "17": 20, @@ -312,159 +286,153 @@ def test_write_tagged_file(fileutils, generator_instance: Generator): "1200": "120", "VALID": "valid", } - cmd_list = generator_instance._write_tagged_files( - files=files, params=param_set, dest=generator_instance.root - ) + to_configure = [ + ConfigureOperation(src=file, dest=dest, file_parameters=file_param) + for file in source + ] + gen = GenerationContext(pathlib.Path(test_dir)) + cmd_list = generator_instance._configure_files(files=to_configure, context=gen) assert isinstance(cmd_list, CommandList) - for file, cmd in zip(tagged_files, cmd_list): - assert file in cmd.command + # Extract file paths from commands + cmd_src_paths = set() + for cmd in cmd_list.commands: + src_index = cmd.command.index("configure") + 1 + cmd_src_paths.add(cmd.command[src_index]) + # Assert all file paths are in the command list + file_paths = {str(file) for file in source} + assert file_paths == cmd_src_paths, "Not all file paths are in the command list" -def test_write_tagged_directory(fileutils, generator_instance: Generator): - """Test Generator._write_tagged_files helper function with directory path""" - config = get_gen_file(fileutils, "tag_dir_template") - files = EntityFiles(tagged=[config]) - param_set = {"PARAM0": "param_value_1", "PARAM1": "param_value_2"} - cmd_list = generator_instance._write_tagged_files( - files=files, params=param_set, dest=generator_instance.root - ) - - assert isinstance(cmd_list, CommandList) - assert str(config) in cmd_list.commands[0].command +@pytest.fixture +def run_directory(test_dir, generator_instance): + return pathlib.Path(test_dir) / generator_instance.run_directory -# INTEGRATED TESTS +@pytest.fixture +def log_directory(test_dir, generator_instance): + return pathlib.Path(test_dir) / generator_instance.log_directory -def test_exp_private_generate_method( - mock_job: unittest.mock.MagicMock, test_dir: str, generator_instance: Generator +def test_build_commands( + generator_instance: Generator, + run_directory: pathlib.Path, + log_directory: pathlib.Path, ): - """Test that Experiment._generate returns expected tuple.""" - mock_index = 1 - exp = Experiment(name="experiment_name", exp_path=test_dir) - job_paths = exp._generate(generator_instance, mock_job, mock_index) - assert osp.isdir(job_paths.run_path) - assert job_paths.out_path.name == f"{mock_job.entity.name}.out" - assert job_paths.err_path.name == f"{mock_job.entity.name}.err" + """Test Generator._build_commands calls internal helper functions""" + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._append_mkdir_commands" + ) as mock_append_mkdir_commands, + unittest.mock.patch( + "smartsim._core.generation.Generator._append_file_operations" + ) as mock_append_file_operations, + ): + generator_instance._build_commands( + EchoHelloWorldEntity(), + run_directory, + log_directory, + ) + mock_append_mkdir_commands.assert_called_once() + mock_append_file_operations.assert_called_once() -def test_generate_ensemble_directory_start( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch -): - """Test that Experiment._generate returns expected tuple.""" - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - ensemble = Ensemble("ensemble-name", "echo", replicas=2) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir_path = pathlib.Path(test_dir) / run_dir[0] / "jobs" - list_of_job_dirs = jobs_dir_path.iterdir() - for job in list_of_job_dirs: - run_path = jobs_dir_path / job / Generator.run_directory - assert run_path.is_dir() - log_path = jobs_dir_path / job / Generator.log_directory - assert log_path.is_dir() - ids.clear() - - -def test_generate_ensemble_copy( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_copy_dir +def test_append_mkdir_commands( + generator_instance: Generator, + run_directory: pathlib.Path, + log_directory: pathlib.Path, ): - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - ensemble = Ensemble( - "ensemble-name", "echo", replicas=2, files=EntityFiles(copy=get_gen_copy_dir) - ) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - job_dir = jobs_dir.iterdir() - for ensemble_dir in job_dir: - copy_folder_path = ( - jobs_dir / ensemble_dir / Generator.run_directory / "to_copy_dir" + """Test Generator._append_mkdir_commands calls Generator._mkdir_file twice""" + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._mkdir_file" + ) as mock_mkdir_file, + ): + generator_instance._append_mkdir_commands( + CommandList(), + run_directory, + log_directory, ) - assert copy_folder_path.is_dir() - ids.clear() + assert mock_mkdir_file.call_count == 2 -def test_generate_ensemble_symlink( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_symlink_dir +def test_append_file_operations( + context: GenerationContext, generator_instance: Generator ): - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - ensemble = Ensemble( - "ensemble-name", - "echo", - replicas=2, - files=EntityFiles(symlink=get_gen_symlink_dir), + """Test Generator._append_file_operations calls all file operations""" + with ( + unittest.mock.patch( + "smartsim._core.generation.Generator._copy_files" + ) as mock_copy_files, + unittest.mock.patch( + "smartsim._core.generation.Generator._symlink_files" + ) as mock_symlink_files, + unittest.mock.patch( + "smartsim._core.generation.Generator._configure_files" + ) as mock_configure_files, + ): + generator_instance._append_file_operations( + CommandList(), + EchoHelloWorldEntity(), + context, + ) + mock_copy_files.assert_called_once() + mock_symlink_files.assert_called_once() + mock_configure_files.assert_called_once() + + +@pytest.fixture +def paths_to_copy(fileutils): + paths = fileutils.get_test_conf_path(osp.join("generator_files", "to_copy_dir")) + yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + + +@pytest.fixture +def paths_to_symlink(fileutils): + paths = fileutils.get_test_conf_path(osp.join("generator_files", "to_symlink_dir")) + yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + + +@pytest.fixture +def paths_to_configure(fileutils): + paths = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") ) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - _ = exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - job_dir = jobs_dir.iterdir() - for ensemble_dir in job_dir: - sym_file_path = pathlib.Path(jobs_dir) / ensemble_dir / "run" / "to_symlink_dir" - assert sym_file_path.is_dir() - assert sym_file_path.is_symlink() - assert os.fspath(sym_file_path.resolve()) == osp.realpath(get_gen_symlink_dir) - ids.clear() - - -def test_generate_ensemble_configure( - test_dir: str, wlmutils, monkeypatch: pytest.MonkeyPatch, get_gen_configure_dir + yield [pathlib.Path(path) for path in sorted(glob(paths + "/*"))] + + +@pytest.fixture +def context(test_dir: str): + yield GenerationContext(pathlib.Path(test_dir)) + + +@pytest.fixture +def operations_list(paths_to_copy, paths_to_symlink, paths_to_configure): + op_list = [] + for file in paths_to_copy: + op_list.append(CopyOperation(src=file)) + for file in paths_to_symlink: + op_list.append(SymlinkOperation(src=file)) + for file in paths_to_configure: + op_list.append(SymlinkOperation(src=file)) + return op_list + + +@pytest.fixture +def formatted_command_list(operations_list: list, context: GenerationContext): + new_list = CommandList() + for file in operations_list: + new_list.append(file.format(context)) + return new_list + + +def test_execute_commands( + operations_list: list, formatted_command_list, generator_instance: Generator ): - monkeypatch.setattr( - "smartsim._core.dispatch._LauncherAdapter.start", - lambda launch, exe, job_execution_path, env, out, err: random_id(), - ) - param_set = {"PARAM0": [0, 1], "PARAM1": [2, 3]} - tagged_files = sorted(glob(get_gen_configure_dir + "/*")) - ensemble = Ensemble( - "ensemble-name", - "echo", - replicas=1, - files=EntityFiles(tagged=tagged_files), - file_parameters=param_set, - ) - launch_settings = LaunchSettings(wlmutils.get_test_launcher()) - job_list = ensemble.build_jobs(launch_settings) - exp = Experiment(name="exp_name", exp_path=test_dir) - _ = exp.start(*job_list) - run_dir = listdir(test_dir) - jobs_dir = pathlib.Path(test_dir) / run_dir[0] / "jobs" - - def _check_generated(param_0, param_1, dir): - assert dir.is_dir() - tagged_0 = dir / "tagged_0.sh" - tagged_1 = dir / "tagged_1.sh" - assert tagged_0.is_file() - assert tagged_1.is_file() - - with open(tagged_0) as f: - line = f.readline() - assert line.strip() == f'echo "Hello with parameter 0 = {param_0}"' - - with open(tagged_1) as f: - line = f.readline() - assert line.strip() == f'echo "Hello with parameter 1 = {param_1}"' - - _check_generated(0, 3, jobs_dir / "ensemble-name-1-1" / Generator.run_directory) - _check_generated(1, 2, jobs_dir / "ensemble-name-2-2" / Generator.run_directory) - _check_generated(1, 3, jobs_dir / "ensemble-name-3-3" / Generator.run_directory) - _check_generated(0, 2, jobs_dir / "ensemble-name-0-0" / Generator.run_directory) - ids.clear() + """Test Generator._execute_commands calls with appropriate type and num times""" + with ( + unittest.mock.patch( + "smartsim._core.generation.generator.subprocess.run" + ) as mock_run, + ): + generator_instance._execute_commands(formatted_command_list) + assert mock_run.call_count == len(formatted_command_list) diff --git a/tests/test_operations.py b/tests/test_operations.py new file mode 100644 index 0000000000..abfc141d89 --- /dev/null +++ b/tests/test_operations.py @@ -0,0 +1,364 @@ +import base64 +import os +import pathlib +import pickle + +import pytest + +from smartsim._core.commands import Command +from smartsim._core.generation.operations.operations import ( + ConfigureOperation, + CopyOperation, + FileSysOperationSet, + GenerationContext, + SymlinkOperation, + _check_run_path, + _create_dest_path, + configure_cmd, + copy_cmd, + default_tag, + symlink_cmd, +) +from smartsim._core.generation.operations.utils.helpers import check_src_and_dest_path + +pytestmark = pytest.mark.group_a + + +@pytest.fixture +def generation_context(test_dir: str): + """Fixture to create a GenerationContext object.""" + return GenerationContext(pathlib.Path(test_dir)) + + +@pytest.fixture +def file_system_operation_set( + copy_operation: CopyOperation, + symlink_operation: SymlinkOperation, + configure_operation: ConfigureOperation, +): + """Fixture to create a FileSysOperationSet object.""" + return FileSysOperationSet([copy_operation, symlink_operation, configure_operation]) + + +# TODO is this test even necessary +@pytest.mark.parametrize( + "job_run_path, dest", + ( + pytest.param( + pathlib.Path("/absolute/src"), + pathlib.Path("relative/dest"), + id="Valid paths", + ), + pytest.param( + pathlib.Path("/absolute/src"), + pathlib.Path(""), + id="Empty destination path", + ), + ), +) +def test_check_src_and_dest_path_valid(job_run_path, dest): + """Test valid path inputs for helpers.check_src_and_dest_path""" + check_src_and_dest_path(job_run_path, dest) + + +@pytest.mark.parametrize( + "job_run_path, dest, error", + ( + pytest.param( + pathlib.Path("relative/src"), + pathlib.Path("relative/dest"), + ValueError, + id="Relative src Path", + ), + pytest.param( + pathlib.Path("/absolute/src"), + pathlib.Path("/absolute/src"), + ValueError, + id="Absolute dest Path", + ), + pytest.param( + 123, + pathlib.Path("relative/dest"), + TypeError, + id="non Path src", + ), + pytest.param( + pathlib.Path("/absolute/src"), + 123, + TypeError, + id="non Path dest", + ), + ), +) +def test_check_src_and_dest_path_invalid(job_run_path, dest, error): + """Test invalid path inputs for helpers.check_src_and_dest_path""" + with pytest.raises(error): + check_src_and_dest_path(job_run_path, dest) + + +@pytest.mark.parametrize( + "job_run_path, dest, expected", + ( + pytest.param( + pathlib.Path("/absolute/root"), + pathlib.Path("relative/dest"), + "/absolute/root/relative/dest", + id="Valid paths", + ), + pytest.param( + pathlib.Path("/absolute/root"), + pathlib.Path(""), + "/absolute/root", + id="Empty destination path", + ), + ), +) +def test_create_dest_path_valid(job_run_path, dest, expected): + """Test valid path inputs for operations._create_dest_path""" + assert _create_dest_path(job_run_path, dest) == expected + + +@pytest.mark.parametrize( + "job_run_path, error", + ( + pytest.param( + pathlib.Path("relative/path"), ValueError, id="Run path is not absolute" + ), + pytest.param(1234, TypeError, id="Run path is not pathlib.path"), + ), +) +def test_check_run_path_invalid(job_run_path, error): + """Test invalid path inputs for operations._check_run_path""" + with pytest.raises(error): + _check_run_path(job_run_path) + + +def test_valid_init_generation_context(test_dir: str): + """Validate GenerationContext init""" + generation_context = GenerationContext(pathlib.Path(test_dir)) + assert isinstance(generation_context, GenerationContext) + assert generation_context.job_run_path == pathlib.Path(test_dir) + + +def test_invalid_init_generation_context(): + """Validate GenerationContext init""" + with pytest.raises(TypeError): + GenerationContext(1234) + with pytest.raises(TypeError): + GenerationContext("") + + +def test_init_copy_operation(mock_src: pathlib.Path, mock_dest: pathlib.Path): + """Validate CopyOperation init""" + copy_operation = CopyOperation(mock_src, mock_dest) + assert isinstance(copy_operation, CopyOperation) + assert copy_operation.src == mock_src + assert copy_operation.dest == mock_dest + + +def test_copy_operation_format( + copy_operation: CopyOperation, + mock_dest: str, + mock_src: str, + generation_context: GenerationContext, + test_dir: str, +): + """Validate CopyOperation.format""" + exec = copy_operation.format(generation_context) + assert isinstance(exec, Command) + assert str(mock_src) in exec.command + assert copy_cmd in exec.command + assert _create_dest_path(test_dir, mock_dest) in exec.command + + +def test_init_symlink_operation(mock_src: str, mock_dest: str): + """Validate SymlinkOperation init""" + symlink_operation = SymlinkOperation(mock_src, mock_dest) + assert isinstance(symlink_operation, SymlinkOperation) + assert symlink_operation.src == mock_src + assert symlink_operation.dest == mock_dest + + +def test_symlink_operation_format( + symlink_operation: SymlinkOperation, + mock_src: str, + mock_dest: str, + generation_context: GenerationContext, +): + """Validate SymlinkOperation.format""" + exec = symlink_operation.format(generation_context) + assert isinstance(exec, Command) + assert str(mock_src) in exec.command + assert symlink_cmd in exec.command + + normalized_path = os.path.normpath(mock_src) + parent_dir = os.path.dirname(normalized_path) + final_dest = _create_dest_path(generation_context.job_run_path, mock_dest) + new_dest = os.path.join(final_dest, parent_dir) + assert new_dest in exec.command + + +def test_init_configure_operation(mock_src: str, mock_dest: str): + """Validate ConfigureOperation init""" + configure_operation = ConfigureOperation( + src=mock_src, dest=mock_dest, file_parameters={"FOO": "BAR"} + ) + assert isinstance(configure_operation, ConfigureOperation) + assert configure_operation.src == mock_src + assert configure_operation.dest == mock_dest + assert configure_operation.tag == default_tag + decoded_dict = base64.b64decode(configure_operation.file_parameters.encode("ascii")) + unpickled_dict = pickle.loads(decoded_dict) + assert unpickled_dict == {"FOO": "BAR"} + + +def test_configure_operation_format( + configure_operation: ConfigureOperation, + test_dir: str, + mock_dest: str, + mock_src: str, + generation_context: GenerationContext, +): + """Validate ConfigureOperation.format""" + exec = configure_operation.format(generation_context) + assert isinstance(exec, Command) + assert str(mock_src) in exec.command + assert configure_cmd in exec.command + assert _create_dest_path(test_dir, mock_dest) in exec.command + + +def test_init_file_sys_operation_set( + copy_operation: CopyOperation, + symlink_operation: SymlinkOperation, + configure_operation: ConfigureOperation, +): + """Test initialize FileSystemOperationSet""" + file_system_operation_set = FileSysOperationSet( + [copy_operation, symlink_operation, configure_operation] + ) + assert isinstance(file_system_operation_set.operations, list) + assert len(file_system_operation_set.operations) == 3 + + +def test_add_copy_operation(file_system_operation_set: FileSysOperationSet): + """Test FileSystemOperationSet.add_copy""" + orig_num_ops = len(file_system_operation_set.copy_operations) + file_system_operation_set.add_copy(src=pathlib.Path("/src")) + assert len(file_system_operation_set.copy_operations) == orig_num_ops + 1 + + +def test_add_symlink_operation(file_system_operation_set: FileSysOperationSet): + """Test FileSystemOperationSet.add_symlink""" + orig_num_ops = len(file_system_operation_set.symlink_operations) + file_system_operation_set.add_symlink(src=pathlib.Path("/src")) + assert len(file_system_operation_set.symlink_operations) == orig_num_ops + 1 + + +def test_add_configure_operation( + file_system_operation_set: FileSysOperationSet, +): + """Test FileSystemOperationSet.add_configuration""" + orig_num_ops = len(file_system_operation_set.configure_operations) + file_system_operation_set.add_configuration( + src=pathlib.Path("/src"), file_parameters={"FOO": "BAR"} + ) + assert len(file_system_operation_set.configure_operations) == orig_num_ops + 1 + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), + ), +) +def test_copy_files_invalid_dest(dest, error, source): + """Test invalid copy destination""" + with pytest.raises(error): + _ = [CopyOperation(src=file, dest=dest) for file in source] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), + ), +) +def test_copy_files_invalid_src(src, error): + """Test invalid copy source""" + with pytest.raises(error): + _ = CopyOperation(src=src) + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), + ), +) +def test_symlink_files_invalid_dest(dest, error, source): + """Test invalid symlink destination""" + with pytest.raises(error): + _ = [SymlinkOperation(src=file, dest=dest) for file in source] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), + ), +) +def test_symlink_files_invalid_src(src, error): + """Test invalid symlink source""" + with pytest.raises(error): + _ = SymlinkOperation(src=src) + + +@pytest.mark.parametrize( + "dest,error", + ( + pytest.param(123, TypeError, id="dest as integer"), + pytest.param("", TypeError, id="dest as empty str"), + pytest.param( + pathlib.Path("/absolute/path"), ValueError, id="dest as absolute str" + ), + ), +) +def test_configure_files_invalid_dest(dest, error, source): + """Test invalid configure destination""" + with pytest.raises(error): + _ = [ + ConfigureOperation(src=file, dest=dest, file_parameters={"FOO": "BAR"}) + for file in source + ] + + +@pytest.mark.parametrize( + "src,error", + ( + pytest.param(123, TypeError, id="src as integer"), + pytest.param("", TypeError, id="src as empty str"), + pytest.param( + pathlib.Path("relative/path"), ValueError, id="src as relative str" + ), + ), +) +def test_configure_files_invalid_src(src, error): + """Test invalid configure source""" + with pytest.raises(error): + _ = ConfigureOperation(src=src, file_parameters={"FOO": "BAR"})