diff --git a/config/opal_check_ofi.m4 b/config/opal_check_ofi.m4 index e873a399fdc..401c8e8d2e9 100644 --- a/config/opal_check_ofi.m4 +++ b/config/opal_check_ofi.m4 @@ -137,7 +137,16 @@ AC_DEFUN([OPAL_CHECK_OFI],[ AC_CHECK_DECLS([PMIX_PACKAGE_RANK], [], [], - [#include ])]) + [#include ]) + + AC_CHECK_MEMBER([struct fi_mr_attr.iface], + [opal_check_fi_mr_attr_iface=1], + [opal_check_fi_mr_attr_iface=0], + [[#include ]]) + + AC_DEFINE_UNQUOTED([OPAL_OFI_HAVE_FI_MR_IFACE], + [${opal_check_fi_mr_attr_iface}], + [check if iface avaiable in fi_mr_attr])]) CPPFLAGS=${opal_check_ofi_save_CPPFLAGS} LDFLAGS=${opal_check_ofi_save_LDFLAGS} diff --git a/ompi/mca/mtl/ofi/mtl_ofi.h b/ompi/mca/mtl/ofi/mtl_ofi.h index 1d7dd0435e8..9b3a8f7dfca 100644 --- a/ompi/mca/mtl/ofi/mtl_ofi.h +++ b/ompi/mca/mtl/ofi/mtl_ofi.h @@ -306,6 +306,8 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor, return OMPI_SUCCESS; } +#if OPAL_OFI_HAVE_FI_MR_IFACE + if ((convertor->flags & CONVERTOR_ACCELERATOR) && ompi_mtl_ofi.hmem_needs_reg) { /* Register buffer */ int ret; @@ -343,6 +345,8 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor, } } +#endif + return OMPI_SUCCESS; } diff --git a/ompi/mca/mtl/ofi/mtl_ofi_component.c b/ompi/mca/mtl/ofi/mtl_ofi_component.c index 98d0537779e..bfd263db202 100644 --- a/ompi/mca/mtl/ofi/mtl_ofi_component.c +++ b/ompi/mca/mtl/ofi/mtl_ofi_component.c @@ -6,7 +6,7 @@ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2018-2022 Amazon.com, Inc. or its affiliates. All Rights reserved. - * Copyright (c) 2020-2021 Triad National Security, LLC. All rights + * Copyright (c) 2020-2023 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -617,8 +617,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads, } /* Request device transfer capabilities */ +#if defined(FI_HMEM) hints->caps |= FI_HMEM; hints->domain_attr->mr_mode |= FI_MR_HMEM | FI_MR_ALLOCATED; +#endif no_hmem: @@ -737,12 +739,14 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads, __FILE__, __LINE__, fi_strerror(-ret)); if (FI_ENODATA == -ret) { +#if defined(FI_HMEM) /* Attempt selecting a provider without FI_HMEM hints */ if (hints->caps & FI_HMEM) { hints->caps &= ~FI_HMEM; hints->domain_attr->mr_mode &= ~FI_MR_HMEM; goto no_hmem; } +#endif /* It is not an error if no information is returned. */ goto error; } else if (0 != ret) { @@ -770,11 +774,12 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads, opal_argv_free(exclude_list); exclude_list = NULL; + *accelerator_support = false; +#if defined(FI_HMEM) if (!(prov->caps & FI_HMEM)) { opal_output_verbose(50, opal_common_ofi.output, "%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n", __FILE__, __LINE__); - *accelerator_support = false; } else { *accelerator_support = true; ompi_mtl_ofi.hmem_needs_reg = true; @@ -791,6 +796,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads, } } +#else + opal_output_verbose(50, opal_common_ofi.output, + "%s:%d: Libfabric provider does not support device buffers. Continuing with device to host copies.\n", + __FILE__, __LINE__); +#endif /** * Select the format of the OFI tag diff --git a/opal/mca/btl/ofi/btl_ofi_component.c b/opal/mca/btl/ofi/btl_ofi_component.c index 579e5facc48..ebf6705be91 100644 --- a/opal/mca/btl/ofi/btl_ofi_component.c +++ b/opal/mca/btl/ofi/btl_ofi_component.c @@ -15,7 +15,7 @@ * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2018-2021 Amazon.com, Inc. or its affiliates. All Rights reserved. - * Copyright (c) 2020-2022 Triad National Security, LLC. All rights + * Copyright (c) 2020-2023 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -106,7 +106,11 @@ static int validate_info(struct fi_info *info, uint64_t required_caps, char **in mr_mode = info->domain_attr->mr_mode; if (!(mr_mode == FI_MR_BASIC || mr_mode == FI_MR_SCALABLE +#if defined(FI_MR_HMEM) || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT | FI_MR_HMEM)) == 0)) { +#else + || (mr_mode & ~(FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_ENDPOINT)) == 0)) { +#endif BTL_VERBOSE(("unsupported MR mode")); return OPAL_ERROR; } @@ -339,20 +343,24 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules, mca_btl_ofi_component.module_count = 0; +#if defined(FI_HMEM) /* Request device transfer capabilities, separate from required_caps */ hints.caps |= FI_HMEM; hints.domain_attr->mr_mode |= FI_MR_HMEM; no_hmem: +#endif /* Do the query. The earliest version that supports FI_HMEM hints is 1.9 */ rc = fi_getinfo(FI_VERSION(1, 9), NULL, NULL, 0, &hints, &info_list); if (0 != rc) { +#if defined(FI_HMEM) if (hints.caps & FI_HMEM) { /* Try again without FI_HMEM hints */ hints.caps &= ~FI_HMEM; hints.domain_attr->mr_mode &= ~FI_MR_HMEM; goto no_hmem; } +#endif BTL_VERBOSE(("fi_getinfo failed with code %d: %s", rc, fi_strerror(-rc))); if (NULL != include_list) { opal_argv_free(include_list); @@ -360,6 +368,7 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules, return NULL; } +#if defined(FI_HMEM) /* If we get to this point with FI_HMEM hint set, we want it to be a * required capability */ @@ -375,6 +384,7 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules, } required_caps |= FI_HMEM; } +#endif /* count the number of resources/ */ info = info_list; @@ -604,9 +614,11 @@ static int mca_btl_ofi_init_device(struct fi_info *info) module->use_fi_mr_bind = false; module->bypass_cache = false; +#if defined(FI_HMEM) if (ofi_info->caps & FI_HMEM) { module->super.btl_flags |= MCA_BTL_FLAGS_ACCELERATOR_RDMA; } +#endif if (ofi_info->domain_attr->mr_mode == FI_MR_BASIC || ofi_info->domain_attr->mr_mode & FI_MR_VIRT_ADDR) { diff --git a/opal/mca/btl/ofi/btl_ofi_module.c b/opal/mca/btl/ofi/btl_ofi_module.c index 8497e559523..ae28dc123a5 100644 --- a/opal/mca/btl/ofi/btl_ofi_module.c +++ b/opal/mca/btl/ofi/btl_ofi_module.c @@ -16,7 +16,7 @@ * * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2020 Google, LLC. All rights reserved. - * Copyright (c) 2022 Triad National Security, LLC. All rights + * Copyright (c) 2022-2023 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -254,6 +254,7 @@ int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size, attr.context = NULL; attr.requested_key = (uint64_t) reg; +#if OPAL_OFI_HAVE_FI_MR_IFACE if (OPAL_LIKELY(NULL != base)) { rc = opal_accelerator.check_addr(base, &dev_id, &flags); if (rc < 0) { @@ -270,6 +271,7 @@ int mca_btl_ofi_reg_mem(void *reg_data, void *base, size_t size, } } } +#endif rc = fi_mr_regattr(btl->domain, &attr, 0, &ur->ur_mr); if (0 != rc) {