diff --git a/benchmarks/case.py b/benchmarks/case.py index be29598..27e8e35 100644 --- a/benchmarks/case.py +++ b/benchmarks/case.py @@ -6,7 +6,7 @@ from jinja2 import Template from settings import * -from utils import read, write +from benchmarks.utils import read, write class Case(object): @@ -25,10 +25,10 @@ def __init__(self, name, type, config, work_dir): self.work_dir = work_dir self.stdout = ".".join((self.name, "log")) self.stdout_file = os.path.join(self.work_dir, self.stdout) - self.config = deepcopy(self._get_default_config()) + self.config = deepcopy(self._get_case_config(config)) self.config.update(config) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. @@ -59,7 +59,7 @@ def _get_rms_context(self): "WALLTIME": self.config["walltime"], "NOTIFY": self.config["notify"], "EMAIL": self.config["email"], - "MODULE": self.config["module"], + "ENVIRONMENT": self.config["environment"], "COMMAND": self.config["command"], "RUNTIME_FILE": RUNTIME_FILE, "CPU_INFO_FILE": CPU_INFO_FILE, diff --git a/benchmarks/espresso/__init__.py b/benchmarks/espresso/__init__.py index a364215..7286933 100644 --- a/benchmarks/espresso/__init__.py +++ b/benchmarks/espresso/__init__.py @@ -1,23 +1,23 @@ import os from benchmarks.case import Case -from settings import ESPRESSO_MODULE +from settings import ESPRESSO_ENVIRONMENT class ESPRESSOCase(Case): def __init__(self, name, type, config, work_dir): super(ESPRESSOCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(ESPRESSOCase, self)._get_default_config() + default_config = super(ESPRESSOCase, self)._get_case_config(config) default_config.update({ - "module": ESPRESSO_MODULE + "environment": ESPRESSO_ENVIRONMENT }) return default_config diff --git a/benchmarks/gromacs/__init__.py b/benchmarks/gromacs/__init__.py index 672c0cc..0561d9d 100644 --- a/benchmarks/gromacs/__init__.py +++ b/benchmarks/gromacs/__init__.py @@ -1,4 +1,4 @@ -from settings import GROMACS_MODULE +from settings import GROMACS_ENVIRONMENT from benchmarks.case import Case, os from benchmarks.utils import read, write @@ -7,16 +7,16 @@ class GROMACSCase(Case): def __init__(self, name, type, config, work_dir): super(GROMACSCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(GROMACSCase, self)._get_default_config() + default_config = super(GROMACSCase, self)._get_case_config(config) default_config.update({ - "module": GROMACS_MODULE + "environment": GROMACS_ENVIRONMENT }) return default_config diff --git a/benchmarks/gromacs/cases.py b/benchmarks/gromacs/cases.py index 219d0ec..1a45641 100644 --- a/benchmarks/gromacs/cases.py +++ b/benchmarks/gromacs/cases.py @@ -1,6 +1,6 @@ import os -from settings import NODES_CONFIGURATION, PPN_CONFIGURATION +from settings import PBS_NP, NODES_CONFIGURATION, PPN_CONFIGURATION GROMACS_CASES = [] CWD = os.path.dirname(__file__) @@ -22,7 +22,7 @@ "template": "benchmarks/gromacs/inputs/{}/md.tpr".format(input_) } ], - "command": "source GMXRC.bash; mpirun -np $PBS_NP gmx_mpi_d mdrun -ntomp 1 -s md.tpr -deffnm md" + "command": f"source GMXRC.bash; mpirun -np {PBS_NP} gmx_mpi_d mdrun -ntomp 1 -s md.tpr -deffnm md" } } GROMACS_CASES.append(config) diff --git a/benchmarks/hpl/__init__.py b/benchmarks/hpl/__init__.py index 0f42d8e..d571e7a 100644 --- a/benchmarks/hpl/__init__.py +++ b/benchmarks/hpl/__init__.py @@ -3,24 +3,25 @@ from benchmarks.case import Case from benchmarks.utils import read -from settings import HPL_MODULE, HPL_RESULT_REGEX +from settings import HPL_ENVIRONMENT, HPL_RESULT_REGEX class HPLCase(Case): def __init__(self, name, type, config, work_dir): super(HPLCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(HPLCase, self)._get_default_config() + mpirun_np = config.get('P', 1) * config.get('Q', 1) + default_config = super(HPLCase, self)._get_case_config(config) default_config.update({ - "module": HPL_MODULE, - "command": "mpirun -np $PBS_NP xhpl &> {}".format(self.stdout), + "environment": HPL_ENVIRONMENT, + "command": f"mpirun -np {mpirun_np} xhpl &> {self.stdout}", "inputs": [ { "name": "HPL.dat", diff --git a/benchmarks/hpl/cases.json b/benchmarks/hpl/cases.json index ae7c14b..d9d69a1 100644 --- a/benchmarks/hpl/cases.json +++ b/benchmarks/hpl/cases.json @@ -1,30 +1,30 @@ [ { "nodes": 1, - "N": 200448, + "N": 6144, "NB": 192, - "P": 6, - "Q": 6 + "P": 1, + "Q": 1 }, { - "nodes": 2, - "N": 283776, + "nodes": 4, + "N": 12288, "NB": 192, - "P": 8, - "Q": 9 + "P": 2, + "Q": 2 }, { - "nodes": 4, - "N": 401280, + "nodes": 9, + "N": 18432, "NB": 192, - "P": 12, - "Q": 12 + "P": 3, + "Q": 3 }, { - "nodes": 8, - "N": 567552, + "nodes": 16, + "N": 24576, "NB": 192, - "P": 16, - "Q": 18 + "P": 4, + "Q": 4 } ] diff --git a/benchmarks/utils.py b/benchmarks/utils.py index 486d866..2d661ff 100644 --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -4,11 +4,28 @@ def read(path): - with open(path) as f: - return f.read() + with open(path, "rb") as f: + # PROBLEM: Not all input file formats are utf-8 + # SOLUTION: Read bytes from file, then attempt to decode to string + file_bytes = f.read() + try: + file_string = file_bytes.decode("utf-8") + return file_string + except UnicodeError: + #print(f"Unable to decode file '{path}' - returning bytes") + pass + return file_bytes def write(path, content, mode="w+"): + # PROBLEM: Not all output file content is utf-8 + # SOLUTION: Encode string as bytes, then write bytes to file + if type(content) == str: + content = content.encode("utf-8") + elif type(content) != bytes: + raise TypeError() + if "b" not in mode: + mode = "b" + mode with open(path, mode) as f: f.write(content) diff --git a/benchmarks/vasp/__init__.py b/benchmarks/vasp/__init__.py index 4e6ba81..7d5aa79 100644 --- a/benchmarks/vasp/__init__.py +++ b/benchmarks/vasp/__init__.py @@ -1,6 +1,6 @@ import os -from settings import VASP_MODULE +from settings import PBS_NP, VASP_ENVIRONMENT from benchmarks.case import Case @@ -8,17 +8,17 @@ class VASPCase(Case): def __init__(self, name, type, config, work_dir): super(VASPCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(VASPCase, self)._get_default_config() + default_config = super(VASPCase, self)._get_case_config(config) default_config.update({ - "module": VASP_MODULE, - "command": "mpirun -np $PBS_NP vasp &> {}".format(self.stdout), + "environment": VASP_ENVIRONMENT, + "command": f"mpirun -np {PBS_NP} vasp &> {self.stdout}", }) return default_config diff --git a/job.rms b/job.rms index 688c0e3..f493151 100644 --- a/job.rms +++ b/job.rms @@ -1,28 +1,27 @@ #!/bin/bash ## Below are resource manager directives. -## Replace them with directives for your system (e.g. #SBATCH), if needed. +## Replace them with directives for your system (e.g. #PBS), if needed. +# https://slurm.schedmd.com/rosetta.pdf -#PBS -N {{ NAME }} -#PBS -j oe -#PBS -R n -#PBS -r n -#PBS -q {{ QUEUE }} -#PBS -l nodes={{ NODES }} -#PBS -l ppn={{ PPN }} -#PBS -l walltime={{ WALLTIME }} -#PBS -m {{ NOTIFY }} -#PBS -M {{ EMAIL }} +#SBATCH --job-name={{ NAME }} +#SBATCH --no-requeue +#SBATCH --partition={{ QUEUE }} +#SBATCH --nodes={{ NODES }} +#SBATCH --ntasks-per-node={{ PPN }} +#SBATCH --time={{ WALLTIME }} +#SBATCH --mail-type {{ NOTIFY }} +#SBATCH --mail-user={{ EMAIL }} -cd $PBS_O_WORKDIR +cd $SLURM_SUBMIT_DIR ## The use of environment modules is assumed below. ## Replace the directive with your system convention if modules are not installed. -module add {{ MODULE }} - ## DO NOT MODIFY LINES BELOW HERE +{{ ENVIRONMENT }} + start=`date +%s` {{ COMMAND }} diff --git a/requirements.txt b/requirements.txt index 46224da..770ec37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -jinja2==2.8 +jinja2==3.1.2 tabulate==0.7.5 requests==2.20.1 -matplotlib==2.2.3 +matplotlib==3.7.1 diff --git a/settings.py b/settings.py index 9fafca7..eb5d1e5 100644 --- a/settings.py +++ b/settings.py @@ -2,11 +2,11 @@ # Name of the site where benchmarks are run. Should not contain space. # Should contain the cloud provider, company and cluster name and should be unique to identify the site. -# Example: "AWS-C5-Exabyte-Cluster007 +# SITE_NAME = "AWS-t2.nano-16nodes" SITE_NAME = "" # Name of the location where the benchmarks are run. Should contain information about the location of the SITE. -# Example: "East-US-zone-a" +# SITE_LOCATION = "us-west-1b" SITE_LOCATION = "" # Name of the files to store the benchmark runtime, CPU and Memory info respectively. @@ -22,37 +22,46 @@ RESULTS_FILE_NAME = "results.json" RESULTS_FILE_PATH = os.path.join(os.path.dirname(__file__), "results", RESULTS_FILE_NAME) -# Whether to publish the results. Set it to False to disable it. -PUBLISH_RESULTS = True +# Whether to publish the results to: +# https://docs.google.com/spreadsheets/d/1oBHR8bp9q86MOxGYXcvUWHZa8KiC-uc_qX-UF1iqf-Y/edit +# Set it to True to enable it. +PUBLISH_RESULTS = False # Google Cloud Function endpoint to send the benchmark results to. REMOTE_SOURCE_API_URL = "https://us-central1-exabyte-io.cloudfunctions.net/Exabyte-Benchmarks-Results" # Resource Management System (RMS) settings for PBS/Torque. Adjust settings to accommodate for other resource managers. # http://docs.adaptivecomputing.com/torque/6-1-0/adminGuide/help.htm#topics/torque/2-jobs/submittingManagingJobs.htm -PPN = 36 -QUEUE = "OF" -NOTIFY = "abe" +PPN = 1 +QUEUE = "queue1" WALLTIME = "05:00:00" +NOTIFY = "BEGIN, END, FAIL" EMAIL = "mohammad@exabyte.io" DEFAULT_RMS_CONFIG = { "NODES": 1, "PPN": PPN, "QUEUE": QUEUE, "WALLTIME": WALLTIME, - "NOTIFY": NOTIFY, - "EMAIL": EMAIL, "RUNTIME_FILE": RUNTIME_FILE, + "NOTIFY": NOTIFY, + "EMAIL": EMAIL } # Name of qsub command -QSUB_COMMAND = "qsub" +QSUB_COMMAND = "sbatch" + +# Argument of -np option of mpirun (computed from P*Q for HPL benchmark) +PBS_NP = 1 -# Environment Modules Settings -HPL_MODULE = "hpl/22-i-174-impi-044" -VASP_MODULE = "vasp/535-i-174-impi-044" -ESPRESSO_MODULE = "espresso/540-i-174-impi-044" -GROMACS_MODULE = "gromacs/20183-i-174-impi-044-md" +# Environment Settings +# HPL_ENVIRONMENT = ''' +# module load intelmpi +# export PATH=$PATH:/home/ubuntu/hpl/bin/linux +# ''' +HPL_ENVIRONMENT = "module add hpl/22-i-174-impi-044" +VASP_ENVIRONMENT = "module add vasp/535-i-174-impi-044" +ESPRESSO_ENVIRONMENT = "module add espresso/540-i-174-impi-044" +GROMACS_ENVIRONMENT = "module add gromacs/20183-i-174-impi-044-md" # Regular expressions to extract the HPL results (N, NB, P, Q, TIME and GFLOPS) REGEX = { @@ -68,6 +77,6 @@ } # Specifies nodes and ppn configurations. -# Cases are generated for all combinations. +# Cases are generated for gromacs and vasp all combinations. NODES_CONFIGURATION = [1, 2, 4, 8] PPN_CONFIGURATION = [4, 8, 12, 16]