Add NVHPC/21.11 and CUDA/11.5.1. (#1392)

* Add + deploy NVIDIA HPC SDK 21.11.
* Add + deploy CUDA 11.5.1 to match it.
* Do not define CMAKE_C_FLAGS in `coreneuron+nmodl`.
* Run `makelocalrc` for `nvhpc` in a cleaner environment.
* Add `-mno-abm` to [Core]NEURON recipes for `[email protected]`.

Co-authored-by: Olli Lupton <[email protected]>

Author: pramodk

deploy/deploy.lib

@@ -339,7 +339,13 @@ configure_compilers() {
             # standard library from the deployed version of GCC.
             NVHPC_DIR=$(dirname $(which makelocalrc))
             NVHPC_TMPDIR=$(mktemp -d -p .)
-            makelocalrc ${NVHPC_DIR} -d "${NVHPC_TMPDIR}" -gcc ${GCC_DIR}/bin/gcc -gpp ${GCC_DIR}/bin/g++ -g77 ${GCC_DIR}/bin/gfortran -x
+            # Try and avoid autoloaded dependencies of other compilers (looking
+            # at you, LLVM) polluting the localrc of the NVIDIA compilers
+            (
+              module purge
+              ${cmd}
+              makelocalrc ${NVHPC_DIR} -d "${NVHPC_TMPDIR}" -gcc ${GCC_DIR}/bin/gcc -gpp ${GCC_DIR}/bin/g++ -g77 ${GCC_DIR}/bin/gfortran -x
+            )
             # olupton 2021-10-15: is this still needed? Not sure.
             echo "set PREOPTIONS=-D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1;" >> "${NVHPC_TMPDIR}/localrc"
             log "NVIDIA localrc template"

deploy/environments/externals.yaml

@@ -63,6 +63,7 @@ spack:
     - [email protected]
     - [email protected] install_type=network
     - [email protected]
+    - [email protected]
     - arm-forge
     - bison
     - blender
@@ -74,6 +75,7 @@ spack:
     - cmake
     - [email protected]
     - [email protected]
+    - [email protected]
     - [email protected]
     - darshan-runtime
     - darshan-util

deploy/set-compiler-flags.py

@@ -0,0 +1,33 @@
+import argparse
+import spack
+from spack.config import config
+
+parser = argparse.ArgumentParser(
+    description="Add C/C++ compiler flags to Spack configuration."
+)
+parser.add_argument(
+    "--scope", default="user", help="Configuration scope to read/write."
+)
+parser.add_argument("--compiler-spec", type=str, help="Compiler spec to modify.")
+parser.add_argument(
+    "--fields",
+    type=str,
+    nargs="+",
+    help="Which type of flags to write.",
+    default=["cflags", "cxxflags"],
+    choices=["cflags", "cxxflags", "cppflags", "fflags", "ldflags", "ldlibs"],
+)
+parser.add_argument(
+    "flags", type=str, nargs="+", help="Compiler flags to set in the configuration."
+)
+args = parser.parse_args()
+flags_str = " ".join(args.flags)
+compiler_configs = config.get("compilers", scope=args.scope)
+for compiler_config in compiler_configs:
+    compiler_spec = spack.spec.Spec(compiler_config["compiler"]["spec"])
+    if compiler_spec.satisfies(args.compiler_spec):
+        compiler_flags = compiler_config["compiler"].get("flags", {})
+        for field in args.fields:
+            compiler_flags[field] = flags_str
+        compiler_config["compiler"]["flags"] = compiler_flags
+config.update_config("compilers", compiler_configs, scope=args.scope)

var/spack/repos/builtin/packages/coreneuron/package.py

@@ -127,6 +127,10 @@ def get_flags(self):
             flags = '-xHost -qopt-report=5'
             if '+knl' in spec:
                 flags = '-xMIC-AVX512 -qopt-report=5'
+        # NVHPC 21.11 detects ABM support and defines __ABM__, which breaks
+        # Random123 compilation
+        if spec.satisfies('%[email protected]'):
+            flags += ' -mno-abm'
         # when pdt is used for instrumentation, the gcc's unint128 extension
         # is activated from random123 which results in compilation error
         if '+profile' in spec:
@@ -143,7 +147,6 @@ def get_cmake_args(self):
 
         options =\
             ['-DCORENRN_ENABLE_SPLAYTREE_QUEUING=ON',
-             '-DCMAKE_C_FLAGS=%s' % flags,
              '-DCMAKE_CXX_FLAGS=%s' % flags,
              '-DCORENRN_ENABLE_REPORTING=%s'
              % ('ON' if '+report' in spec else 'OFF'),
@@ -156,6 +159,11 @@ def get_cmake_args(self):
              '-DPYTHON_EXECUTABLE=%s' % spec["python"].command.path
              ]
 
+        # Versions after this only used C++, but we might still need C
+        # flags if mod2c is being built as a submodule.
+        if spec.satisfies('@:1.0.0.20210708') or spec.satisfies('~nmodl'):
+            options.append('-DCMAKE_C_FLAGS=%s' % flags)
+
         if spec.satisfies('+caliper'):
             options.append('-DCORENRN_ENABLE_CALIPER_PROFILING=ON')
 

var/spack/repos/builtin/packages/cuda/package.py

@@ -25,6 +25,10 @@
 #    format returned by platform.system() and 'arch' by platform.machine()
 
 _versions = {
+    '11.5.1': {
+        'Linux-aarch64': ('73e1d0e97c7fa686efe7e00fb1e5f179372c4eec8e14d4f44ab58d5f6cf57f63', 'https://developer.download.nvidia.com/compute/cuda/11.5.1/local_installers/cuda_11.5.1_495.29.05_linux_sbsa.run'),
+        'Linux-x86_64': ('60bea2fc0fac95574015f865355afbf599422ec2c85554f5f052b292711a4bca', 'https://developer.download.nvidia.com/compute/cuda/11.5.1/local_installers/cuda_11.5.1_495.29.05_linux.run'),
+        'Linux-ppc64le': ('9e0e494d945634fe8ad3e12d7b91806aa4220ed27487bb211030d651b27c67a9', 'https://developer.download.nvidia.com/compute/cuda/11.5.1/local_installers/cuda_11.5.1_495.29.05_linux_ppc64le.run')},
     '11.5.0': {
         'Linux-aarch64': ('6ea9d520cc956cc751a5ac54f4acc39109627f4e614dd0b1a82cc86f2aa7d8c4', 'https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux_sbsa.run'),
         'Linux-x86_64': ('ae0a1693d9497cf3d81e6948943e3794636900db71c98d58eefdacaf7f1a1e4c', 'https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux.run'),

var/spack/repos/builtin/packages/neuron/package.py

@@ -143,6 +143,7 @@ def cmake_enable_option(spec_requiremement):
                                                              "+rx3d",
                                                              "+coreneuron",
                                                              "+tests"]]
+        compilation_flags = []
         if self.spec.variants['model_tests'].value != ("None",):
             args.append('-DNRN_ENABLE_MODEL_TESTS=' + ",".join(
                 model for model in self.spec.variants["model_tests"].value))
@@ -156,8 +157,8 @@ def cmake_enable_option(spec_requiremement):
             args.append("-DPYTHON_EXECUTABLE:FILEPATH="
                         + self.spec["python"].command.path)
         if "+debug" in self.spec:
-            args.append("-DCMAKE_C_FLAGS=-g -O0")
-            args.append("-DCMAKE_CXX_FLAGS=-g -O0")
+            compilation_flags += ['-g', '-O0']
+            # Remove default flags (RelWithDebInfo etc.)
             args.append("-DCMAKE_BUILD_TYPE=Custom")
         if "+mod-compatibility" in self.spec:
             args.append("-DNRN_ENABLE_MOD_COMPATIBILITY:BOOL=ON")
@@ -167,6 +168,13 @@ def cmake_enable_option(spec_requiremement):
             args.append('-DNRN_DYNAMIC_UNITS_USE_LEGACY=ON')
         if "+coreneuron" in self.spec:
             args.append('-DCORENEURON_DIR=' + self.spec["coreneuron"].prefix)
+        # NVHPC 21.11 detects ABM support and defines __ABM__, which breaks
+        # Random123 compilation
+        if self.spec.satisfies('%[email protected]'):
+            compilation_flags.append('-mno-abm')
+        compilation_flags = ' '.join(compilation_flags)
+        args.append("-DCMAKE_C_FLAGS=" + compilation_flags)
+        args.append("-DCMAKE_CXX_FLAGS=" + compilation_flags)
         return args
 
     # Create symlink in share/nrn/lib for the python libraries

var/spack/repos/builtin/packages/nvhpc/package.py

@@ -22,6 +22,10 @@
 #  - package key must be in the form '{os}-{arch}' where 'os' is in the
 #    format returned by platform.system() and 'arch' by platform.machine()
 _versions = {
+    '21.11': {
+        'Linux-aarch64': ('3b11bcd9cca862fabfce1e7bcaa2050ea12130c7e897f4e7859ba4c155d20720', 'https://developer.download.nvidia.com/hpc-sdk/21.11/nvhpc_2021_2111_Linux_aarch64_cuda_multi.tar.gz'),
+        'Linux-ppc64le': ('ac51ed92de4eb5e1bdb064ada5bbace5b89ac732ad6c6473778edfb8d29a6527', 'https://developer.download.nvidia.com/hpc-sdk/21.11/nvhpc_2021_2111_Linux_ppc64le_cuda_multi.tar.gz'),
+        'Linux-x86_64': ('d8d8ccd0e558d22bcddd955f2233219c96f7de56aa8e09e7be833e384d32d6aa', 'https://developer.download.nvidia.com/hpc-sdk/21.11/nvhpc_2021_2111_Linux_x86_64_cuda_multi.tar.gz')},
     '21.9': {
         'Linux-aarch64': ('52c2c66e30043add4afccedf0ba77daa0000bf42e0db844baa630bb635b91a7d', 'https://developer.download.nvidia.com/hpc-sdk/21.9/nvhpc_2021_219_Linux_aarch64_cuda_multi.tar.gz'),
         'Linux-ppc64le': ('cff0b55fb782be1982bfeec1d9763b674ddbf84ff2c16b364495299266320289', 'https://developer.download.nvidia.com/hpc-sdk/21.9/nvhpc_2021_219_Linux_ppc64le_cuda_multi.tar.gz'),