diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 4c0a07423a..b3c66d3734 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -42,6 +42,7 @@ * Caspar van Leeuwen (SURF) * Jan Andre Reuter (Juelich Supercomputing Centre) * Jasper Grimm (UoY) +* Alex Domingo (Vrije Universiteit Brussel) """ import concurrent import copy @@ -2964,9 +2965,21 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True): self.modules_tool.load(extra_modules) # Setup CUDA cache if required. If we don't do this, CUDA will use the $HOME for its cache files - if get_software_root('CUDA') or get_software_root('CUDAcore'): + if get_software_root('CUDA') or get_software_root('CUDAcore') or get_software_root('nvidia-compilers'): self.set_up_cuda_cache() + # Set CUDA compute capabilities from default value in nvidia-compiler/NVHPC toolchains + if get_software_root('nvidia-compilers'): + cuda_cc_cfg = self.cfg.get('cuda_compute_capabilities') + cuda_cc_opt = build_option('cuda_compute_capabilities') + cuda_cc_nvhpc = os.getenv('EBNVHPCCUDACC', None) + if not cuda_cc_cfg and not cuda_cc_opt and cuda_cc_nvhpc: + self.cfg['cuda_compute_capabilities'] = cuda_cc_nvhpc.split(',') + self.log.info( + "Updated empty 'cuda_compute_capabilities' option with default CUDA compute capability " + f"defined in nvidia-compilers: {self.cfg['cuda_compute_capabilities']}" + ) + # guess directory to start configure/build/install process in, and move there if start_dir: self.guess_start_dir() diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 46fe4c5970..b056858bb3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -33,14 +33,14 @@ * Fotis Georgatos (Uni.Lu, NTUA) * Kenneth Hoste (Ghent University) """ -import re +import os import platform +import re from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.systemtools import get_shared_lib_ext, pick_dep_version from easybuild.tools.config import build_option - +from easybuild.tools.systemtools import get_shared_lib_ext, pick_dep_version _log = fancylogger.getLogger('easyconfig.templates', fname=False) @@ -85,6 +85,10 @@ 'Python': 'py', 'R': 'r', } +TEMPLATE_CUDA_VERSION_NVHPC = [ + 'nvidia_compilers', + 'NVHPC', +] # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = { 'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)', @@ -92,6 +96,7 @@ "--cuda-compute-capabilities configuration option or " "via cuda_compute_capabilities easyconfig parameter", 'cuda_cc_cmake': 'List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+', + 'cuda_cc_nvhpc': 'List of CUDA compute capabilities suitable for use with -gpu option in NVHPC compilers', 'cuda_cc_space_sep': 'Space-separated list of CUDA compute capabilities', 'cuda_cc_space_sep_no_period': "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90').", @@ -424,6 +429,17 @@ def template_constant_dict(config, ignore=None, toolchain=None): template_values['%sminver' % pref] = dep_version_parts[1] template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) + # step 2.1: CUDA templates in NVHPC + if toolchain is not None and hasattr(toolchain, 'name') and toolchain.name in TEMPLATE_CUDA_VERSION_NVHPC: + cuda_version = os.getenv("EBNVHPCCUDAVER", None) + if cuda_version: + cuda_version_parts = cuda_version.split('.') + template_values['cudaver'] = cuda_version + template_values['cudamajver'] = cuda_version_parts[0] + if len(cuda_version_parts) > 1: + template_values['cudaminver'] = cuda_version_parts[1] + template_values['cudashortver'] = '.'.join(cuda_version_parts[:2]) + # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: if name in ignore: @@ -470,6 +486,7 @@ def template_constant_dict(config, ignore=None, toolchain=None): template_values['cuda_cc_space_sep_no_period'] = ' '.join(cc.replace('.', '') for cc in cuda_cc) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_cc) template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_cc) + template_values['cuda_cc_nvhpc'] = ','.join(f"cc{cc.replace('.', '')}" for cc in cuda_cc) int_values = [cc.replace('.', '') for cc in cuda_cc] template_values['cuda_int_comma_sep'] = ','.join(int_values) template_values['cuda_int_space_sep'] = ' '.join(int_values) diff --git a/easybuild/toolchains/compiler/nvhpc.py b/easybuild/toolchains/compiler/nvidia_compilers.py similarity index 97% rename from easybuild/toolchains/compiler/nvhpc.py rename to easybuild/toolchains/compiler/nvidia_compilers.py index 1821afff73..2cbe493fce 100644 --- a/easybuild/toolchains/compiler/nvhpc.py +++ b/easybuild/toolchains/compiler/nvidia_compilers.py @@ -39,15 +39,13 @@ import easybuild.tools.systemtools as systemtools from easybuild.tools.toolchain.compiler import Compiler - TC_CONSTANT_NVHPC = "NVHPC" -class NVHPC(Compiler): - """NVHPC compiler class - """ +class NvidiaCompilers(Compiler): + "NVHPC compiler class" - COMPILER_MODULE_NAME = ['NVHPC'] + COMPILER_MODULE_NAME = ['nvidia-compilers'] COMPILER_FAMILY = TC_CONSTANT_NVHPC diff --git a/easybuild/toolchains/linalg/nvblas.py b/easybuild/toolchains/linalg/nvblas.py new file mode 100644 index 0000000000..c036949caf --- /dev/null +++ b/easybuild/toolchains/linalg/nvblas.py @@ -0,0 +1,49 @@ +## +# Copyright 2013-2025 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for BLAS and LAPACK libraries in NVHPC as toolchain linear algebra library. + +Authors: + +* Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.tools.toolchain.linalg import LinAlg + + +class NVBLAS(LinAlg): + """ + NVIDIA HPC SDK distributes its own BLAS and LAPACK libraries based on a custom OpenBLAS + see https://docs.nvidia.com/hpc-sdk/compilers/hpc-compilers-user-guide/index.html#lapack-blas-and-ffts + """ + BLAS_MODULE_NAME = ['NVHPC'] + BLAS_LIB = ['blas'] + BLAS_LIB_MT = ['blas'] + BLAS_FAMILY = 'OpenBLAS' + + LAPACK_IS_BLAS = False + LAPACK_MODULE_NAME = ['NVHPC'] + LAPACK_LIB = ['lapack'] + LAPACK_FAMILY = 'LAPACK' diff --git a/easybuild/toolchains/linalg/nvscalapack.py b/easybuild/toolchains/linalg/nvscalapack.py new file mode 100644 index 0000000000..1150d3da4b --- /dev/null +++ b/easybuild/toolchains/linalg/nvscalapack.py @@ -0,0 +1,57 @@ +## +# Copyright 2012-2025 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for ScaLAPACK libraries in NVHPC as toolchain linear algebra library. + +Authors: + +* Alex Domingo (Vrije Universiteit Brussel) +""" + +from easybuild.tools.toolchain.linalg import LinAlg + + +class NVScaLAPACK(LinAlg): + """ + NVIDIA HPC SDK distributes its own ScaLAPACK libraries + see https://docs.nvidia.com/hpc-sdk/compilers/hpc-compilers-user-guide/index.html#linking-with-scalapack + """ + # the following emulates the `-Mscalapack` macro as defined in $NVHPC/compilers/bin/rcfiles/lin86rc + SCALAPACK_MODULE_NAME = ['NVHPC'] + SCALAPACK_LIB_MAP = {'lp64_sc': '_lp64'} + SCALAPACK_LIB = ["scalapack%(lp64_sc)s", "lapack%(lp64_sc)s", "blas%(lp64_sc)s"] + SCALAPACK_LIB_MT = ["scalapack%(lp64_sc)s", "lapack%(lp64_sc)s", "blas%(lp64_sc)s"] + SCALAPACK_REQUIRES = ['LIBLAPACK', 'LIBBLAS'] + + def _set_scalapack_variables(self): + if self.options.get('i8', None): + # ilp64/i8 + self.SCALAPACK_LIB_MAP.update({"lp64_sc": '_ilp64'}) + + super()._set_scalapack_variables() + + def _set_blacs_variables(self): + """Skip setting BLACS related variables""" + pass diff --git a/easybuild/toolchains/mpi/nvhpcx.py b/easybuild/toolchains/mpi/nvhpcx.py new file mode 100644 index 0000000000..e73467882b --- /dev/null +++ b/easybuild/toolchains/mpi/nvhpcx.py @@ -0,0 +1,55 @@ +## +# Copyright 2012-2025 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Support for NVHPCX as toolchain MPI library. + +Authors: + +* Alex Domingo (Vrije Universiteit Brussel) +""" +from easybuild.toolchains.mpi.openmpi import OpenMPI +from easybuild.tools.toolchain.constants import MPI_COMPILER_VARIABLES + +TC_CONSTANT_OPENMPI = "OpenMPI" +TC_CONSTANT_MPI_TYPE_OPENMPI = "MPI_TYPE_OPENMPI" + + +class NVHPCX(OpenMPI): + """NVHPCX MPI class""" + MPI_MODULE_NAME = ['NVHPC'] + MPI_FAMILY = TC_CONSTANT_OPENMPI + MPI_TYPE = TC_CONSTANT_MPI_TYPE_OPENMPI + + MPI_LIBRARY_NAME = 'mpi' + + # version-dependent, so defined at runtime + MPI_COMPILER_MPIF77 = None + MPI_COMPILER_MPIF90 = None + MPI_COMPILER_MPIFC = None + + # OpenMPI reads from CC etc env variables + MPI_SHARED_OPTION_MAP = {'_opt_%s' % var: '' for var, _ in MPI_COMPILER_VARIABLES} + + MPI_LINK_INFO_OPTION = '-showme:link' diff --git a/easybuild/toolchains/nvhpc.py b/easybuild/toolchains/nvhpc.py index 9a0db2bbd7..c7a67574a5 100644 --- a/easybuild/toolchains/nvhpc.py +++ b/easybuild/toolchains/nvhpc.py @@ -1,14 +1,11 @@ ## -# Copyright 2015-2025 Bart Oldeman -# -# This file is triple-licensed under GPLv2 (see below), MIT, and -# BSD three-clause licenses. +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), # with support of Ghent University (http://ugent.be/hpc), # the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# the Hercules foundation (http://www.herculesstichting.be/in_English) +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). # # https://github.com/easybuilders/easybuild @@ -26,23 +23,21 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for NVHPC compiler toolchain. +EasyBuild support for NVHPC compiler toolchain with support for MPI Authors: * Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) * Andreas Herten (Forschungszentrum Juelich) +* Alex Domingo (Vrije Universiteit Brussel) """ - -from easybuild.toolchains.compiler.nvhpc import NVHPC -from easybuild.toolchains.gcccore import GCCcore -from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME +from easybuild.toolchains.linalg.nvblas import NVBLAS +from easybuild.toolchains.linalg.nvscalapack import NVScaLAPACK +from easybuild.toolchains.mpi.nvhpcx import NVHPCX +from easybuild.toolchains.nvidia_compilers import NvidiaCompilersToolchain -class NVHPCToolchain(NVHPC): - """Simple toolchain with just the NVIDIA HPC SDK compilers.""" +class NVHPC(NvidiaCompilersToolchain, NVHPCX, NVBLAS, NVScaLAPACK): + """Toolchain with Nvidia compilers and NVHPCX.""" NAME = 'NVHPC' - # use GCCcore as subtoolchain rather than GCC, since two 'real' compiler-only toolchains don't mix well, - # in particular in a hierarchical module naming scheme - SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME] - OPTIONAL = False + SUBTOOLCHAIN = NvidiaCompilersToolchain.NAME diff --git a/easybuild/toolchains/nvidia_compilers.py b/easybuild/toolchains/nvidia_compilers.py new file mode 100644 index 0000000000..68de03cda5 --- /dev/null +++ b/easybuild/toolchains/nvidia_compilers.py @@ -0,0 +1,48 @@ +## +# Copyright 2015-2025 Bart Oldeman +# +# This file is triple-licensed under GPLv2 (see below), MIT, and +# BSD three-clause licenses. +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for NVHPC compiler toolchain. + +Authors: + +* Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +* Andreas Herten (Forschungszentrum Juelich) +""" + +from easybuild.toolchains.compiler.nvidia_compilers import NvidiaCompilers +from easybuild.toolchains.gcccore import GCCcore +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME + + +class NvidiaCompilersToolchain(NvidiaCompilers): + """Simple toolchain with just the NVIDIA HPC SDK compilers.""" + NAME = 'nvidia-compilers' + # use GCCcore as subtoolchain rather than GCC, since two 'real' compiler-only toolchains don't mix well, + # in particular in a hierarchical module naming scheme + SUBTOOLCHAIN = [GCCcore.NAME, SYSTEM_TOOLCHAIN_NAME] + OPTIONAL = False diff --git a/easybuild/toolchains/nvofbf.py b/easybuild/toolchains/nvofbf.py index 06bc8fda64..8a19bf48da 100644 --- a/easybuild/toolchains/nvofbf.py +++ b/easybuild/toolchains/nvofbf.py @@ -29,10 +29,10 @@ :author: Mikael Öhman (Chalmers University of Technology) """ -from easybuild.toolchains.nvompi import Nvompi from easybuild.toolchains.fft.fftw import Fftw from easybuild.toolchains.linalg.flexiblas import FlexiBLAS from easybuild.toolchains.linalg.scalapack import ScaLAPACK +from easybuild.toolchains.nvompi import Nvompi class Nvofbf(Nvompi, FlexiBLAS, ScaLAPACK, Fftw): diff --git a/easybuild/toolchains/nvompi.py b/easybuild/toolchains/nvompi.py index 578d3f9262..ca5faa269d 100644 --- a/easybuild/toolchains/nvompi.py +++ b/easybuild/toolchains/nvompi.py @@ -30,11 +30,11 @@ * Robert Mijakovic (LuxProvide) """ -from easybuild.toolchains.nvhpc import NVHPCToolchain from easybuild.toolchains.mpi.openmpi import OpenMPI +from easybuild.toolchains.nvidia_compilers import NvidiaCompilersToolchain -class Nvompi(NVHPCToolchain, OpenMPI): +class Nvompi(NvidiaCompilersToolchain, OpenMPI): """Compiler toolchain with NVHPC and Open MPI.""" NAME = 'nvompi' - SUBTOOLCHAIN = NVHPCToolchain.NAME + SUBTOOLCHAIN = NvidiaCompilersToolchain.NAME diff --git a/easybuild/toolchains/nvompic.py b/easybuild/toolchains/nvompic.py index 648cefc874..73f97e6b4b 100644 --- a/easybuild/toolchains/nvompic.py +++ b/easybuild/toolchains/nvompic.py @@ -32,14 +32,14 @@ * Sebastian Achilles (Forschungszentrum Juelich) """ -from easybuild.toolchains.nvhpc import NVHPCToolchain # We pull in MPI and CUDA at once so this maps nicely to HMNS -from easybuild.toolchains.mpi.openmpi import OpenMPI from easybuild.toolchains.compiler.cuda import Cuda +from easybuild.toolchains.mpi.openmpi import OpenMPI +from easybuild.toolchains.nvidia_compilers import NvidiaCompilersToolchain # Order matters! -class NVompic(NVHPCToolchain, Cuda, OpenMPI): +class NVompic(NvidiaCompilersToolchain, Cuda, OpenMPI): """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency.""" NAME = 'nvompic' - SUBTOOLCHAIN = NVHPCToolchain.NAME + SUBTOOLCHAIN = NvidiaCompilersToolchain.NAME diff --git a/easybuild/toolchains/nvpsmpi.py b/easybuild/toolchains/nvpsmpi.py index 8eb03cca89..37c9cad48e 100644 --- a/easybuild/toolchains/nvpsmpi.py +++ b/easybuild/toolchains/nvpsmpi.py @@ -31,12 +31,12 @@ * Robert Mijakovic (LuxProvide) """ -from easybuild.toolchains.nvhpc import NVHPCToolchain from easybuild.toolchains.mpi.psmpi import Psmpi +from easybuild.toolchains.nvhpc import NvidiaCompilersToolchain # Order matters! -class NVpsmpi(NVHPCToolchain, Psmpi): +class NVpsmpi(NvidiaCompilersToolchain, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI.""" NAME = 'nvpsmpi' - SUBTOOLCHAIN = NVHPCToolchain.NAME + SUBTOOLCHAIN = NvidiaCompilersToolchain.NAME diff --git a/easybuild/toolchains/nvpsmpic.py b/easybuild/toolchains/nvpsmpic.py index 90a7d63257..f50aeacb49 100644 --- a/easybuild/toolchains/nvpsmpic.py +++ b/easybuild/toolchains/nvpsmpic.py @@ -32,14 +32,14 @@ * Sebastian Achilles (Forschungszentrum Juelich) """ -from easybuild.toolchains.nvhpc import NVHPCToolchain # We pull in MPI and CUDA at once so this maps nicely to HMNS -from easybuild.toolchains.mpi.psmpi import Psmpi from easybuild.toolchains.compiler.cuda import Cuda +from easybuild.toolchains.mpi.psmpi import Psmpi +from easybuild.toolchains.nvhpc import NvidiaCompilersToolchain # Order matters! -class NVpsmpic(NVHPCToolchain, Cuda, Psmpi): +class NVpsmpic(NvidiaCompilersToolchain, Cuda, Psmpi): """Compiler toolchain with NVHPC and ParaStationMPI, with CUDA as dependency.""" NAME = 'nvpsmpic' - SUBTOOLCHAIN = NVHPCToolchain.NAME + SUBTOOLCHAIN = NvidiaCompilersToolchain.NAME diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 2f501d2fec..9623cc2212 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -71,6 +71,7 @@ def get_mpi_cmd_template(mpi_family, params, mpi_version=None): toolchain.MPICH: mpirun_n_cmd, toolchain.MPICH2: mpirun_n_cmd, toolchain.MPITRAMPOLINE: "mpiexec -n %(nr_ranks)s %(cmd)s", + toolchain.NVHPC: mpirun_n_cmd, } # Intel MPI mpirun needs more work