ofi: support RHEL7 and libfabric pre 1.9

Fix to allow Open MPI to build on RHEL7 systems with default distro libfabric.

Related to #10954

Signed-off-by: Howard Pritchard <[email protected]>

Author: hppritcha

config/opal_check_ofi.m4

@@ -137,7 +137,16 @@ AC_DEFUN([OPAL_CHECK_OFI],[
            AC_CHECK_DECLS([PMIX_PACKAGE_RANK],
                           [],
                           [],
-                          [#include <pmix.h>])])
+                          [#include <pmix.h>])
+
+           AC_CHECK_MEMBER([struct fi_mr_attr.iface],
+                           [opal_check_fi_mr_attr_iface=1],
+                           [opal_check_fi_mr_attr_iface=0],
+                           [[#include <rdma/fi_domain.h>]])
+
+           AC_DEFINE_UNQUOTED([OPAL_OFI_HAVE_FI_MR_IFACE],
+                              [${opal_check_fi_mr_attr_iface}],
+                              [check if iface avaiable in fi_mr_attr])])
 
     CPPFLAGS=${opal_check_ofi_save_CPPFLAGS}
     LDFLAGS=${opal_check_ofi_save_LDFLAGS}

ompi/mca/mtl/ofi/mtl_ofi.h

@@ -306,6 +306,8 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
         return OMPI_SUCCESS;
     }
 
+#if OPAL_OFI_HAVE_FI_MR_IFACE
+
     if ((convertor->flags & CONVERTOR_ACCELERATOR) && ompi_mtl_ofi.hmem_needs_reg) {
         /* Register buffer */
         int ret;
@@ -343,6 +345,8 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
         }
     }
 
+#endif
+
     return OMPI_SUCCESS;
 }
 

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -6,7 +6,7 @@
  * Copyright (c) 2015-2016 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2020-2021 Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2023 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -617,8 +617,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     }
 
     /* Request device transfer capabilities */
+#if defined(FI_HMEM)
     hints->caps |= FI_HMEM;
     hints->domain_attr->mr_mode |= FI_MR_HMEM | FI_MR_ALLOCATED;
+#endif
 
 no_hmem:
 
@@ -737,12 +739,14 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
                         __FILE__, __LINE__, fi_strerror(-ret));
 
     if (FI_ENODATA == -ret) {
+#if defined(FI_HMEM)
         /* Attempt selecting a provider without FI_HMEM hints */
         if (hints->caps & FI_HMEM) {
             hints->caps &= ~FI_HMEM;
             hints->domain_attr->mr_mode &= ~FI_MR_HMEM;
             goto no_hmem;
         }
+#endif
         /* It is not an error if no information is returned. */
         goto error;
     } else if (0 != ret) {
@@ -770,11 +774,12 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     opal_argv_free(exclude_list);
     exclude_list = NULL;
 
+    *accelerator_support = false;
+#if defined(FI_HMEM)
     if (!(prov->caps & FI_HMEM)) {
         opal_output_verbose(50, opal_common_ofi.output,
                             "%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n",
                             __FILE__, __LINE__);
-        *accelerator_support = false;
     } else {
         *accelerator_support = true;
         ompi_mtl_ofi.hmem_needs_reg = true;
@@ -791,6 +796,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
          }
 
     }
+#else
+    opal_output_verbose(50, opal_common_ofi.output,
+                        "%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n",
+                        __FILE__, __LINE__);
+#endif
 
     /**
      * Select the format of the OFI tag

opal/mca/btl/ofi/btl_ofi_component.c

@@ -15,7 +15,7 @@
  * Copyright (c) 2018-2019 Intel, Inc.  All rights reserved.
  *
  * Copyright (c) 2018-2021 Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2020-2022 Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2023 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -106,7 +106,11 @@ static int validate_info(struct fi_info *info, uint64_t required_caps, char **in
     mr_mode = info->domain_attr->mr_mode;
 
     if (!(mr_mode == FI_MR_BASIC || mr_mode == FI_MR_SCALABLE
+#if defined(FI_MR_HMEM)
           || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT | FI_MR_HMEM)) == 0)) {
+#else
+          || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT)) == 0)) {
+#endif
         BTL_VERBOSE(("unsupported MR mode"));
         return OPAL_ERROR;
     }
@@ -339,27 +343,32 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,
 
     mca_btl_ofi_component.module_count = 0;
 
+#if defined(FI_HMEM)
     /* Request device transfer capabilities, separate from required_caps */
     hints.caps |= FI_HMEM;
     hints.domain_attr->mr_mode |= FI_MR_HMEM;
 no_hmem:
+#endif
 
     /* Do the query. The earliest version that supports FI_HMEM hints is 1.9  */
     rc = fi_getinfo(FI_VERSION(1, 9), NULL, NULL, 0, &hints, &info_list);
     if (0 != rc) {
+#if defined(FI_HMEM)
         if (hints.caps & FI_HMEM) {
             /* Try again without FI_HMEM hints */
             hints.caps &= ~FI_HMEM;
             hints.domain_attr->mr_mode &= ~FI_MR_HMEM;
             goto no_hmem;
         }
+#endif
         BTL_VERBOSE(("fi_getinfo failed with code %d: %s", rc, fi_strerror(-rc)));
         if (NULL != include_list) {
             opal_argv_free(include_list);
         }
         return NULL;
     }
 
+#if defined(FI_HMEM)
     /* If we get to this point with FI_HMEM hint set, we want it to be a
      * required capability
      */
@@ -375,6 +384,7 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,
         }
         required_caps |= FI_HMEM;
     }
+#endif
 
     /* count the number of resources/ */
     info = info_list;
@@ -604,9 +614,11 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     module->use_fi_mr_bind = false;
     module->bypass_cache = false;
 
+#if defined(FI_HMEM)
     if (ofi_info->caps & FI_HMEM) {
         module->super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_RDMA;
     }
+#endif
 
     if (ofi_info->domain_attr->mr_mode == FI_MR_BASIC
         || ofi_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) {

opal/mca/btl/ofi/btl_ofi_module.c

@@ -16,7 +16,7 @@
  *
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2020      Google, LLC. All rights reserved.
- * Copyright (c) 2022      Triad National Security, LLC. All rights
+ * Copyright (c) 2022-2023 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -254,6 +254,7 @@ int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size,
     attr.context = NULL;
     attr.requested_key = (uint64_t) reg;
 
+#if  OPAL_OFI_HAVE_FI_MR_IFACE
     if (OPAL_LIKELY(NULL != base)) {
         rc = opal_accelerator.check_addr(base, &dev_id, &flags);
         if (rc < 0) {
@@ -270,6 +271,7 @@ int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size,
             }
         }
     }
+#endif
 
     rc = fi_mr_regattr(btl->domain, &attr, 0, &ur->ur_mr);
     if (0 != rc) {