From ebb6772d263f8bc2a36e26ed864dc34847bff798 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 11 Apr 2023 18:28:30 -0700 Subject: [PATCH 01/18] Fix: update jinja2 to lastest version This avoids execution error "ImportError: cannot import name 'soft_unicode' from 'markupsafe'" --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 46224da..de5bb2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -jinja2==2.8 +jinja2==3.1.2 tabulate==0.7.5 requests==2.20.1 matplotlib==2.2.3 From d80dae6a6de9c6f229f6e47e53e86a3a718e1845 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 11 Apr 2023 21:35:55 -0700 Subject: [PATCH 02/18] Complete benchmarks.utils import path --- benchmarks/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/case.py b/benchmarks/case.py index be29598..2e274fd 100644 --- a/benchmarks/case.py +++ b/benchmarks/case.py @@ -6,7 +6,7 @@ from jinja2 import Template from settings import * -from utils import read, write +from benchmarks.utils import read, write class Case(object): From e2cc18eb3c281e534092af4aa1e3dd8d5f52872d Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 11 Apr 2023 22:34:36 -0700 Subject: [PATCH 03/18] Generalize utils read and write to handle non utf-8 data as bytes --- benchmarks/utils.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/benchmarks/utils.py b/benchmarks/utils.py index 486d866..2d661ff 100644 --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -4,11 +4,28 @@ def read(path): - with open(path) as f: - return f.read() + with open(path, "rb") as f: + # PROBLEM: Not all input file formats are utf-8 + # SOLUTION: Read bytes from file, then attempt to decode to string + file_bytes = f.read() + try: + file_string = file_bytes.decode("utf-8") + return file_string + except UnicodeError: + #print(f"Unable to decode file '{path}' - returning bytes") + pass + return file_bytes def write(path, content, mode="w+"): + # PROBLEM: Not all output file content is utf-8 + # SOLUTION: Encode string as bytes, then write bytes to file + if type(content) == str: + content = content.encode("utf-8") + elif type(content) != bytes: + raise TypeError() + if "b" not in mode: + mode = "b" + mode with open(path, mode) as f: f.write(content) From 10e1c528158d8b4c6b5d98a7ebfb46b465b28593 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 11 Apr 2023 23:15:44 -0700 Subject: [PATCH 04/18] Fix: update matplotlib to latest version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index de5bb2d..770ec37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ jinja2==3.1.2 tabulate==0.7.5 requests==2.20.1 -matplotlib==2.2.3 +matplotlib==3.7.1 From 129db7951c31c9a85afd696f8ace0d92887c64d6 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Sat, 15 Apr 2023 22:37:42 -0700 Subject: [PATCH 05/18] Remove unknown -R option --- job.rms | 1 - 1 file changed, 1 deletion(-) diff --git a/job.rms b/job.rms index 688c0e3..8e2e40e 100644 --- a/job.rms +++ b/job.rms @@ -5,7 +5,6 @@ #PBS -N {{ NAME }} #PBS -j oe -#PBS -R n #PBS -r n #PBS -q {{ QUEUE }} #PBS -l nodes={{ NODES }} From 0d37d7b3159dd25ca299d6a5e1ab1a6083922be2 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 11 Apr 2023 23:15:59 -0700 Subject: [PATCH 06/18] Disable publishing to google doc --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 9fafca7..3aecb5d 100644 --- a/settings.py +++ b/settings.py @@ -23,7 +23,7 @@ RESULTS_FILE_PATH = os.path.join(os.path.dirname(__file__), "results", RESULTS_FILE_NAME) # Whether to publish the results. Set it to False to disable it. -PUBLISH_RESULTS = True +PUBLISH_RESULTS = False # Google Cloud Function endpoint to send the benchmark results to. REMOTE_SOURCE_API_URL = "https://us-central1-exabyte-io.cloudfunctions.net/Exabyte-Benchmarks-Results" From b33b6a70dc5e7cea44bd5dddd86b283aaa693e8d Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Sat, 15 Apr 2023 22:39:36 -0700 Subject: [PATCH 07/18] TEMP: Customize settings for GabrielHare --- settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/settings.py b/settings.py index 3aecb5d..231cfba 100644 --- a/settings.py +++ b/settings.py @@ -3,11 +3,11 @@ # Name of the site where benchmarks are run. Should not contain space. # Should contain the cloud provider, company and cluster name and should be unique to identify the site. # Example: "AWS-C5-Exabyte-Cluster007 -SITE_NAME = "" +SITE_NAME = "AWS-t2.nano-16nodes" # Name of the location where the benchmarks are run. Should contain information about the location of the SITE. # Example: "East-US-zone-a" -SITE_LOCATION = "" +SITE_LOCATION = "us-west-1b" # Name of the files to store the benchmark runtime, CPU and Memory info respectively. RUNTIME_FILE = "runtime" @@ -31,7 +31,7 @@ # Resource Management System (RMS) settings for PBS/Torque. Adjust settings to accommodate for other resource managers. # http://docs.adaptivecomputing.com/torque/6-1-0/adminGuide/help.htm#topics/torque/2-jobs/submittingManagingJobs.htm PPN = 36 -QUEUE = "OF" +QUEUE = "queue1" NOTIFY = "abe" WALLTIME = "05:00:00" EMAIL = "mohammad@exabyte.io" From e95f2017b75433f0aa9750d0aaa48ff0e471b02e Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Sun, 16 Apr 2023 22:13:22 -0700 Subject: [PATCH 08/18] Convert from PBS to SLURM --- job.rms | 21 ++++++++++----------- settings.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/job.rms b/job.rms index 8e2e40e..e6f15a1 100644 --- a/job.rms +++ b/job.rms @@ -3,17 +3,16 @@ ## Below are resource manager directives. ## Replace them with directives for your system (e.g. #SBATCH), if needed. -#PBS -N {{ NAME }} -#PBS -j oe -#PBS -r n -#PBS -q {{ QUEUE }} -#PBS -l nodes={{ NODES }} -#PBS -l ppn={{ PPN }} -#PBS -l walltime={{ WALLTIME }} -#PBS -m {{ NOTIFY }} -#PBS -M {{ EMAIL }} - -cd $PBS_O_WORKDIR +#SBATCH --job-name={{ NAME }} +#SBATCH --no-requeue +#SBATCH -p {{ QUEUE }} +#SBATCH -N {{ NODES }} +#SBATCH -n {{ PPN }} +#SBATCH -t {{ WALLTIME }} +#SBATCH -mail-type={{ NOTIFY }} +#SBATCH -mail-user={{ EMAIL }} + +cd $SLURM_SUBMIT_DIR ## The use of environment modules is assumed below. ## Replace the directive with your system convention if modules are not installed. diff --git a/settings.py b/settings.py index 231cfba..a2106b3 100644 --- a/settings.py +++ b/settings.py @@ -46,7 +46,7 @@ } # Name of qsub command -QSUB_COMMAND = "qsub" +QSUB_COMMAND = "sbatch" # Environment Modules Settings HPL_MODULE = "hpl/22-i-174-impi-044" From 5cf9e6e214187fcb6c13c713b7323b574795fe04 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Sun, 16 Apr 2023 22:45:21 -0700 Subject: [PATCH 09/18] Replace unknown environment variable PBS_NP with setting --- benchmarks/gromacs/cases.py | 4 ++-- benchmarks/hpl/__init__.py | 4 ++-- benchmarks/vasp/__init__.py | 4 ++-- settings.py | 3 +++ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/benchmarks/gromacs/cases.py b/benchmarks/gromacs/cases.py index 219d0ec..1a45641 100644 --- a/benchmarks/gromacs/cases.py +++ b/benchmarks/gromacs/cases.py @@ -1,6 +1,6 @@ import os -from settings import NODES_CONFIGURATION, PPN_CONFIGURATION +from settings import PBS_NP, NODES_CONFIGURATION, PPN_CONFIGURATION GROMACS_CASES = [] CWD = os.path.dirname(__file__) @@ -22,7 +22,7 @@ "template": "benchmarks/gromacs/inputs/{}/md.tpr".format(input_) } ], - "command": "source GMXRC.bash; mpirun -np $PBS_NP gmx_mpi_d mdrun -ntomp 1 -s md.tpr -deffnm md" + "command": f"source GMXRC.bash; mpirun -np {PBS_NP} gmx_mpi_d mdrun -ntomp 1 -s md.tpr -deffnm md" } } GROMACS_CASES.append(config) diff --git a/benchmarks/hpl/__init__.py b/benchmarks/hpl/__init__.py index 0f42d8e..c7842cc 100644 --- a/benchmarks/hpl/__init__.py +++ b/benchmarks/hpl/__init__.py @@ -3,7 +3,7 @@ from benchmarks.case import Case from benchmarks.utils import read -from settings import HPL_MODULE, HPL_RESULT_REGEX +from settings import PBS_NP, HPL_MODULE, HPL_RESULT_REGEX class HPLCase(Case): @@ -20,7 +20,7 @@ def _get_default_config(self): default_config = super(HPLCase, self)._get_default_config() default_config.update({ "module": HPL_MODULE, - "command": "mpirun -np $PBS_NP xhpl &> {}".format(self.stdout), + "command": f"mpirun -np {PBS_NP} xhpl &> {self.stdout}", "inputs": [ { "name": "HPL.dat", diff --git a/benchmarks/vasp/__init__.py b/benchmarks/vasp/__init__.py index 4e6ba81..c5030ac 100644 --- a/benchmarks/vasp/__init__.py +++ b/benchmarks/vasp/__init__.py @@ -1,6 +1,6 @@ import os -from settings import VASP_MODULE +from settings import PBS_NP, VASP_MODULE from benchmarks.case import Case @@ -18,7 +18,7 @@ def _get_default_config(self): default_config = super(VASPCase, self)._get_default_config() default_config.update({ "module": VASP_MODULE, - "command": "mpirun -np $PBS_NP vasp &> {}".format(self.stdout), + "command": f"mpirun -np {PBS_NP} vasp &> {self.stdout}", }) return default_config diff --git a/settings.py b/settings.py index a2106b3..396d2c1 100644 --- a/settings.py +++ b/settings.py @@ -48,6 +48,9 @@ # Name of qsub command QSUB_COMMAND = "sbatch" +# Argument of -np option of mpirun +PBS_NP = 1 + # Environment Modules Settings HPL_MODULE = "hpl/22-i-174-impi-044" VASP_MODULE = "vasp/535-i-174-impi-044" From 7fd931b0798433b554cdabee8aa2b09bbd458850 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Mon, 17 Apr 2023 14:38:14 -0700 Subject: [PATCH 10/18] Use full SLURM command line flag names for clarity --- job.rms | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/job.rms b/job.rms index e6f15a1..b85892a 100644 --- a/job.rms +++ b/job.rms @@ -5,12 +5,12 @@ #SBATCH --job-name={{ NAME }} #SBATCH --no-requeue -#SBATCH -p {{ QUEUE }} -#SBATCH -N {{ NODES }} -#SBATCH -n {{ PPN }} -#SBATCH -t {{ WALLTIME }} -#SBATCH -mail-type={{ NOTIFY }} -#SBATCH -mail-user={{ EMAIL }} +#SBATCH --partition={{ QUEUE }} +#SBATCH --nodes={{ NODES }} +#SBATCH --ntasks-per-node={{ PPN }} +#SBATCH --time={{ WALLTIME }} +# #SBATCH --mail-type={{ NOTIFY }} +# #SBATCH --mail-user={{ EMAIL }} cd $SLURM_SUBMIT_DIR From 89b829e845ccefc711d0cf7db5a12ac46aca7960 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Mon, 17 Apr 2023 14:49:55 -0700 Subject: [PATCH 11/18] Reduce ppn to 1 for consistency with t2.micro --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 396d2c1..ff712b0 100644 --- a/settings.py +++ b/settings.py @@ -30,7 +30,7 @@ # Resource Management System (RMS) settings for PBS/Torque. Adjust settings to accommodate for other resource managers. # http://docs.adaptivecomputing.com/torque/6-1-0/adminGuide/help.htm#topics/torque/2-jobs/submittingManagingJobs.htm -PPN = 36 +PPN = 1 QUEUE = "queue1" NOTIFY = "abe" WALLTIME = "05:00:00" From 6312684fc2ee489f90c198338fe07b2ea72fc56e Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Mon, 17 Apr 2023 23:00:39 -0700 Subject: [PATCH 12/18] Generalize environment changes to add module or append to path --- job.rms | 4 ++-- settings.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/job.rms b/job.rms index b85892a..4a8b35d 100644 --- a/job.rms +++ b/job.rms @@ -17,10 +17,10 @@ cd $SLURM_SUBMIT_DIR ## The use of environment modules is assumed below. ## Replace the directive with your system convention if modules are not installed. -module add {{ MODULE }} - ## DO NOT MODIFY LINES BELOW HERE +{{ MODULE }} + start=`date +%s` {{ COMMAND }} diff --git a/settings.py b/settings.py index ff712b0..f09a750 100644 --- a/settings.py +++ b/settings.py @@ -51,11 +51,11 @@ # Argument of -np option of mpirun PBS_NP = 1 -# Environment Modules Settings -HPL_MODULE = "hpl/22-i-174-impi-044" -VASP_MODULE = "vasp/535-i-174-impi-044" -ESPRESSO_MODULE = "espresso/540-i-174-impi-044" -GROMACS_MODULE = "gromacs/20183-i-174-impi-044-md" +# Environment Settings +HPL_MODULE = "module add hpl/22-i-174-impi-044" +VASP_MODULE = "module add vasp/535-i-174-impi-044" +ESPRESSO_MODULE = "module add espresso/540-i-174-impi-044" +GROMACS_MODULE = "module add gromacs/20183-i-174-impi-044-md" # Regular expressions to extract the HPL results (N, NB, P, Q, TIME and GFLOPS) REGEX = { From b48209950da48c61ccfd0b17388677e278b92b98 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Mon, 17 Apr 2023 23:01:19 -0700 Subject: [PATCH 13/18] Append HPL build location to path --- settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/settings.py b/settings.py index f09a750..227221b 100644 --- a/settings.py +++ b/settings.py @@ -52,7 +52,8 @@ PBS_NP = 1 # Environment Settings -HPL_MODULE = "module add hpl/22-i-174-impi-044" +HPL_MODULE = "export PATH=$PATH:/home/ubuntu/hpl/bin/linux" +#HPL_MODULE = "module add hpl/22-i-174-impi-044" VASP_MODULE = "module add vasp/535-i-174-impi-044" ESPRESSO_MODULE = "module add espresso/540-i-174-impi-044" GROMACS_MODULE = "module add gromacs/20183-i-174-impi-044-md" From 73a48b6a97b951f51175724bf0acb212cedd141e Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 18 Apr 2023 14:52:58 -0700 Subject: [PATCH 14/18] Configure for cluster of 16 single core instances --- benchmarks/hpl/cases.json | 30 +++++++++++++++--------------- settings.py | 6 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/benchmarks/hpl/cases.json b/benchmarks/hpl/cases.json index ae7c14b..d9d69a1 100644 --- a/benchmarks/hpl/cases.json +++ b/benchmarks/hpl/cases.json @@ -1,30 +1,30 @@ [ { "nodes": 1, - "N": 200448, + "N": 6144, "NB": 192, - "P": 6, - "Q": 6 + "P": 1, + "Q": 1 }, { - "nodes": 2, - "N": 283776, + "nodes": 4, + "N": 12288, "NB": 192, - "P": 8, - "Q": 9 + "P": 2, + "Q": 2 }, { - "nodes": 4, - "N": 401280, + "nodes": 9, + "N": 18432, "NB": 192, - "P": 12, - "Q": 12 + "P": 3, + "Q": 3 }, { - "nodes": 8, - "N": 567552, + "nodes": 16, + "N": 24576, "NB": 192, - "P": 16, - "Q": 18 + "P": 4, + "Q": 4 } ] diff --git a/settings.py b/settings.py index 227221b..38e4910 100644 --- a/settings.py +++ b/settings.py @@ -48,8 +48,8 @@ # Name of qsub command QSUB_COMMAND = "sbatch" -# Argument of -np option of mpirun -PBS_NP = 1 +# Argument of -np option of mpirun - for hpl this must be equal to maximum P*Q +PBS_NP = 16 # Environment Settings HPL_MODULE = "export PATH=$PATH:/home/ubuntu/hpl/bin/linux" @@ -72,6 +72,6 @@ } # Specifies nodes and ppn configurations. -# Cases are generated for all combinations. +# Cases are generated for gromacs and vasp all combinations. NODES_CONFIGURATION = [1, 2, 4, 8] PPN_CONFIGURATION = [4, 8, 12, 16] From ae1c0a9ed399324edd65bbdce2eb84a0c6ed4cbd Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 18 Apr 2023 15:20:31 -0700 Subject: [PATCH 15/18] Ensure that impi will be used (HPL build expects this) --- job.rms | 2 ++ 1 file changed, 2 insertions(+) diff --git a/job.rms b/job.rms index 4a8b35d..3b0999b 100644 --- a/job.rms +++ b/job.rms @@ -19,6 +19,8 @@ cd $SLURM_SUBMIT_DIR ## DO NOT MODIFY LINES BELOW HERE +module load intelmpi + {{ MODULE }} start=`date +%s` From 26d3edea1bfb6e80cc556ac4666b06c10b8c1408 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 18 Apr 2023 17:04:22 -0700 Subject: [PATCH 16/18] Refactor case configuration accept arguments --- benchmarks/case.py | 4 ++-- benchmarks/espresso/__init__.py | 4 ++-- benchmarks/gromacs/__init__.py | 4 ++-- benchmarks/hpl/__init__.py | 4 ++-- benchmarks/vasp/__init__.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/case.py b/benchmarks/case.py index 2e274fd..e030533 100644 --- a/benchmarks/case.py +++ b/benchmarks/case.py @@ -25,10 +25,10 @@ def __init__(self, name, type, config, work_dir): self.work_dir = work_dir self.stdout = ".".join((self.name, "log")) self.stdout_file = os.path.join(self.work_dir, self.stdout) - self.config = deepcopy(self._get_default_config()) + self.config = deepcopy(self._get_case_config(config)) self.config.update(config) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. diff --git a/benchmarks/espresso/__init__.py b/benchmarks/espresso/__init__.py index a364215..02b1acc 100644 --- a/benchmarks/espresso/__init__.py +++ b/benchmarks/espresso/__init__.py @@ -8,14 +8,14 @@ class ESPRESSOCase(Case): def __init__(self, name, type, config, work_dir): super(ESPRESSOCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(ESPRESSOCase, self)._get_default_config() + default_config = super(ESPRESSOCase, self)._get_case_config(config) default_config.update({ "module": ESPRESSO_MODULE }) diff --git a/benchmarks/gromacs/__init__.py b/benchmarks/gromacs/__init__.py index 672c0cc..8ddd326 100644 --- a/benchmarks/gromacs/__init__.py +++ b/benchmarks/gromacs/__init__.py @@ -7,14 +7,14 @@ class GROMACSCase(Case): def __init__(self, name, type, config, work_dir): super(GROMACSCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(GROMACSCase, self)._get_default_config() + default_config = super(GROMACSCase, self)._get_case_config(config) default_config.update({ "module": GROMACS_MODULE }) diff --git a/benchmarks/hpl/__init__.py b/benchmarks/hpl/__init__.py index c7842cc..563f4c7 100644 --- a/benchmarks/hpl/__init__.py +++ b/benchmarks/hpl/__init__.py @@ -10,14 +10,14 @@ class HPLCase(Case): def __init__(self, name, type, config, work_dir): super(HPLCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(HPLCase, self)._get_default_config() + default_config = super(HPLCase, self)._get_case_config(config) default_config.update({ "module": HPL_MODULE, "command": f"mpirun -np {PBS_NP} xhpl &> {self.stdout}", diff --git a/benchmarks/vasp/__init__.py b/benchmarks/vasp/__init__.py index c5030ac..ba846fb 100644 --- a/benchmarks/vasp/__init__.py +++ b/benchmarks/vasp/__init__.py @@ -8,14 +8,14 @@ class VASPCase(Case): def __init__(self, name, type, config, work_dir): super(VASPCase, self).__init__(name, type, config, work_dir) - def _get_default_config(self): + def _get_case_config(self, config): """ Returns a default config for the case. It will be merged by the passed config. Returns: dict """ - default_config = super(VASPCase, self)._get_default_config() + default_config = super(VASPCase, self)._get_case_config(config) default_config.update({ "module": VASP_MODULE, "command": f"mpirun -np {PBS_NP} vasp &> {self.stdout}", From 14d6ab1738ac28fae27e879f8d252b5523e33e77 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Tue, 18 Apr 2023 17:16:49 -0700 Subject: [PATCH 17/18] For HPL set mpi -np to P*Q --- benchmarks/hpl/__init__.py | 5 +++-- settings.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/hpl/__init__.py b/benchmarks/hpl/__init__.py index 563f4c7..d31785a 100644 --- a/benchmarks/hpl/__init__.py +++ b/benchmarks/hpl/__init__.py @@ -3,7 +3,7 @@ from benchmarks.case import Case from benchmarks.utils import read -from settings import PBS_NP, HPL_MODULE, HPL_RESULT_REGEX +from settings import HPL_MODULE, HPL_RESULT_REGEX class HPLCase(Case): @@ -17,10 +17,11 @@ def _get_case_config(self, config): Returns: dict """ + mpirun_np = config.get('P', 1) * config.get('Q', 1) default_config = super(HPLCase, self)._get_case_config(config) default_config.update({ "module": HPL_MODULE, - "command": f"mpirun -np {PBS_NP} xhpl &> {self.stdout}", + "command": f"mpirun -np {mpirun_np} xhpl &> {self.stdout}", "inputs": [ { "name": "HPL.dat", diff --git a/settings.py b/settings.py index 38e4910..714abc4 100644 --- a/settings.py +++ b/settings.py @@ -48,7 +48,7 @@ # Name of qsub command QSUB_COMMAND = "sbatch" -# Argument of -np option of mpirun - for hpl this must be equal to maximum P*Q +# Argument of -np option of mpirun PBS_NP = 16 # Environment Settings From 6187c08adc97356e76cfaeea402bea20c7d3c679 Mon Sep 17 00:00:00 2001 From: Gabriel Hare Date: Fri, 21 Apr 2023 01:33:14 -0700 Subject: [PATCH 18/18] Generalize settings and job files to handle environment configuration --- benchmarks/case.py | 2 +- benchmarks/espresso/__init__.py | 4 ++-- benchmarks/gromacs/__init__.py | 4 ++-- benchmarks/hpl/__init__.py | 4 ++-- benchmarks/vasp/__init__.py | 4 ++-- job.rms | 11 +++++------ settings.py | 35 +++++++++++++++++++-------------- 7 files changed, 34 insertions(+), 30 deletions(-) diff --git a/benchmarks/case.py b/benchmarks/case.py index e030533..27e8e35 100644 --- a/benchmarks/case.py +++ b/benchmarks/case.py @@ -59,7 +59,7 @@ def _get_rms_context(self): "WALLTIME": self.config["walltime"], "NOTIFY": self.config["notify"], "EMAIL": self.config["email"], - "MODULE": self.config["module"], + "ENVIRONMENT": self.config["environment"], "COMMAND": self.config["command"], "RUNTIME_FILE": RUNTIME_FILE, "CPU_INFO_FILE": CPU_INFO_FILE, diff --git a/benchmarks/espresso/__init__.py b/benchmarks/espresso/__init__.py index 02b1acc..7286933 100644 --- a/benchmarks/espresso/__init__.py +++ b/benchmarks/espresso/__init__.py @@ -1,7 +1,7 @@ import os from benchmarks.case import Case -from settings import ESPRESSO_MODULE +from settings import ESPRESSO_ENVIRONMENT class ESPRESSOCase(Case): @@ -17,7 +17,7 @@ def _get_case_config(self, config): """ default_config = super(ESPRESSOCase, self)._get_case_config(config) default_config.update({ - "module": ESPRESSO_MODULE + "environment": ESPRESSO_ENVIRONMENT }) return default_config diff --git a/benchmarks/gromacs/__init__.py b/benchmarks/gromacs/__init__.py index 8ddd326..0561d9d 100644 --- a/benchmarks/gromacs/__init__.py +++ b/benchmarks/gromacs/__init__.py @@ -1,4 +1,4 @@ -from settings import GROMACS_MODULE +from settings import GROMACS_ENVIRONMENT from benchmarks.case import Case, os from benchmarks.utils import read, write @@ -16,7 +16,7 @@ def _get_case_config(self, config): """ default_config = super(GROMACSCase, self)._get_case_config(config) default_config.update({ - "module": GROMACS_MODULE + "environment": GROMACS_ENVIRONMENT }) return default_config diff --git a/benchmarks/hpl/__init__.py b/benchmarks/hpl/__init__.py index d31785a..d571e7a 100644 --- a/benchmarks/hpl/__init__.py +++ b/benchmarks/hpl/__init__.py @@ -3,7 +3,7 @@ from benchmarks.case import Case from benchmarks.utils import read -from settings import HPL_MODULE, HPL_RESULT_REGEX +from settings import HPL_ENVIRONMENT, HPL_RESULT_REGEX class HPLCase(Case): @@ -20,7 +20,7 @@ def _get_case_config(self, config): mpirun_np = config.get('P', 1) * config.get('Q', 1) default_config = super(HPLCase, self)._get_case_config(config) default_config.update({ - "module": HPL_MODULE, + "environment": HPL_ENVIRONMENT, "command": f"mpirun -np {mpirun_np} xhpl &> {self.stdout}", "inputs": [ { diff --git a/benchmarks/vasp/__init__.py b/benchmarks/vasp/__init__.py index ba846fb..7d5aa79 100644 --- a/benchmarks/vasp/__init__.py +++ b/benchmarks/vasp/__init__.py @@ -1,6 +1,6 @@ import os -from settings import PBS_NP, VASP_MODULE +from settings import PBS_NP, VASP_ENVIRONMENT from benchmarks.case import Case @@ -17,7 +17,7 @@ def _get_case_config(self, config): """ default_config = super(VASPCase, self)._get_case_config(config) default_config.update({ - "module": VASP_MODULE, + "environment": VASP_ENVIRONMENT, "command": f"mpirun -np {PBS_NP} vasp &> {self.stdout}", }) return default_config diff --git a/job.rms b/job.rms index 3b0999b..f493151 100644 --- a/job.rms +++ b/job.rms @@ -1,7 +1,8 @@ #!/bin/bash ## Below are resource manager directives. -## Replace them with directives for your system (e.g. #SBATCH), if needed. +## Replace them with directives for your system (e.g. #PBS), if needed. +# https://slurm.schedmd.com/rosetta.pdf #SBATCH --job-name={{ NAME }} #SBATCH --no-requeue @@ -9,8 +10,8 @@ #SBATCH --nodes={{ NODES }} #SBATCH --ntasks-per-node={{ PPN }} #SBATCH --time={{ WALLTIME }} -# #SBATCH --mail-type={{ NOTIFY }} -# #SBATCH --mail-user={{ EMAIL }} +#SBATCH --mail-type {{ NOTIFY }} +#SBATCH --mail-user={{ EMAIL }} cd $SLURM_SUBMIT_DIR @@ -19,9 +20,7 @@ cd $SLURM_SUBMIT_DIR ## DO NOT MODIFY LINES BELOW HERE -module load intelmpi - -{{ MODULE }} +{{ ENVIRONMENT }} start=`date +%s` diff --git a/settings.py b/settings.py index 714abc4..eb5d1e5 100644 --- a/settings.py +++ b/settings.py @@ -2,12 +2,12 @@ # Name of the site where benchmarks are run. Should not contain space. # Should contain the cloud provider, company and cluster name and should be unique to identify the site. -# Example: "AWS-C5-Exabyte-Cluster007 -SITE_NAME = "AWS-t2.nano-16nodes" +# SITE_NAME = "AWS-t2.nano-16nodes" +SITE_NAME = "" # Name of the location where the benchmarks are run. Should contain information about the location of the SITE. -# Example: "East-US-zone-a" -SITE_LOCATION = "us-west-1b" +# SITE_LOCATION = "us-west-1b" +SITE_LOCATION = "" # Name of the files to store the benchmark runtime, CPU and Memory info respectively. RUNTIME_FILE = "runtime" @@ -22,7 +22,9 @@ RESULTS_FILE_NAME = "results.json" RESULTS_FILE_PATH = os.path.join(os.path.dirname(__file__), "results", RESULTS_FILE_NAME) -# Whether to publish the results. Set it to False to disable it. +# Whether to publish the results to: +# https://docs.google.com/spreadsheets/d/1oBHR8bp9q86MOxGYXcvUWHZa8KiC-uc_qX-UF1iqf-Y/edit +# Set it to True to enable it. PUBLISH_RESULTS = False # Google Cloud Function endpoint to send the benchmark results to. @@ -32,31 +34,34 @@ # http://docs.adaptivecomputing.com/torque/6-1-0/adminGuide/help.htm#topics/torque/2-jobs/submittingManagingJobs.htm PPN = 1 QUEUE = "queue1" -NOTIFY = "abe" WALLTIME = "05:00:00" +NOTIFY = "BEGIN, END, FAIL" EMAIL = "mohammad@exabyte.io" DEFAULT_RMS_CONFIG = { "NODES": 1, "PPN": PPN, "QUEUE": QUEUE, "WALLTIME": WALLTIME, - "NOTIFY": NOTIFY, - "EMAIL": EMAIL, "RUNTIME_FILE": RUNTIME_FILE, + "NOTIFY": NOTIFY, + "EMAIL": EMAIL } # Name of qsub command QSUB_COMMAND = "sbatch" -# Argument of -np option of mpirun -PBS_NP = 16 +# Argument of -np option of mpirun (computed from P*Q for HPL benchmark) +PBS_NP = 1 # Environment Settings -HPL_MODULE = "export PATH=$PATH:/home/ubuntu/hpl/bin/linux" -#HPL_MODULE = "module add hpl/22-i-174-impi-044" -VASP_MODULE = "module add vasp/535-i-174-impi-044" -ESPRESSO_MODULE = "module add espresso/540-i-174-impi-044" -GROMACS_MODULE = "module add gromacs/20183-i-174-impi-044-md" +# HPL_ENVIRONMENT = ''' +# module load intelmpi +# export PATH=$PATH:/home/ubuntu/hpl/bin/linux +# ''' +HPL_ENVIRONMENT = "module add hpl/22-i-174-impi-044" +VASP_ENVIRONMENT = "module add vasp/535-i-174-impi-044" +ESPRESSO_ENVIRONMENT = "module add espresso/540-i-174-impi-044" +GROMACS_ENVIRONMENT = "module add gromacs/20183-i-174-impi-044-md" # Regular expressions to extract the HPL results (N, NB, P, Q, TIME and GFLOPS) REGEX = {