From 3aca4af548a3d781b6b52f89f4d6c7e66d379609 Mon Sep 17 00:00:00 2001 From: Michael Heinz Date: Mon, 23 Sep 2019 13:09:09 -0400 Subject: [PATCH] REF6976 Silent failure of OMPI over OFI with large messages sizes INTERNAL: STL-59403 The OFI (libfabric) MTL does not respect the maximum message size parameter that OFI provides in the fi_info data. This patch adds this missing max_msg_size field to the mca_ofi_module_t structure and adds a length check to the low-level send routines. Change-Id: I05aa71d332f2df897133b30c28bf37d98f061996 Signed-off-by: Michael Heinz Reviewed-by: Adam Goldman Reviewed-by: Brendan Cunningham --- ompi/mca/mtl/ofi/help-mtl-ofi.txt | 2 ++ ompi/mca/mtl/ofi/mtl_ofi.h | 20 ++++++++++++++++++-- ompi/mca/mtl/ofi/mtl_ofi_component.c | 3 ++- ompi/mca/mtl/ofi/mtl_ofi_types.h | 3 +++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/ompi/mca/mtl/ofi/help-mtl-ofi.txt b/ompi/mca/mtl/ofi/help-mtl-ofi.txt index c6257867749..56778f63b53 100644 --- a/ompi/mca/mtl/ofi/help-mtl-ofi.txt +++ b/ompi/mca/mtl/ofi/help-mtl-ofi.txt @@ -75,3 +75,5 @@ recoverable and your application is likely to abort. Local host: %s Remote host: %s Error: %s (%d) +[message too big] +Message size %llu bigger than supported by selected transport. Max = %llu diff --git a/ompi/mca/mtl/ofi/mtl_ofi.h b/ompi/mca/mtl/ofi/mtl_ofi.h index 56f560f9172..9301f6d6412 100644 --- a/ompi/mca/mtl/ofi/mtl_ofi.h +++ b/ompi/mca/mtl/ofi/mtl_ofi.h @@ -421,13 +421,22 @@ ompi_mtl_ofi_send_generic(struct mca_mtl_base_module_t *mtl, sep_peer_fiaddr = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits); ompi_ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after); - if (OMPI_SUCCESS != ompi_ret) return ompi_ret; + if (OPAL_UNLIKELY(OMPI_SUCCESS != ompi_ret)) { + return ompi_ret; + } ofi_req.buffer = (free_after) ? start : NULL; ofi_req.length = length; ofi_req.status.MPI_ERROR = OMPI_SUCCESS; ofi_req.completion_count = 0; + if (OPAL_UNLIKELY(length > endpoint->mtl_ofi_module->max_msg_size)) { + opal_show_help("help-mtl-ofi.txt", + "message too big", false, + length, endpoint->mtl_ofi_module->max_msg_size); + return OMPI_ERROR; + } + if (ofi_cq_data) { match_bits = mtl_ofi_create_send_tag_CQD(comm->c_contextid, tag); src_addr = sep_peer_fiaddr; @@ -553,13 +562,20 @@ ompi_mtl_ofi_isend_generic(struct mca_mtl_base_module_t *mtl, sep_peer_fiaddr = fi_rx_addr(endpoint->peer_fiaddr, ctxt_id, ompi_mtl_ofi.rx_ctx_bits); ompi_ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after); - if (OMPI_SUCCESS != ompi_ret) return ompi_ret; + if (OMPI_UNLIKELY(OMPI_SUCCESS != ompi_ret)) return ompi_ret; ofi_req->buffer = (free_after) ? start : NULL; ofi_req->length = length; ofi_req->status.MPI_ERROR = OMPI_SUCCESS; ofi_req->completion_count = 1; + if (OPAL_UNLIKELY(length > endpoint->mtl_ofi_module->max_msg_size)) { + opal_show_help("help-mtl-ofi.txt", + "message too big", false, + length, endpoint->mtl_ofi_module->max_msg_size); + return OMPI_ERROR; + } + if (ofi_cq_data) { match_bits = mtl_ofi_create_send_tag_CQD(comm->c_contextid, tag); } else { diff --git a/ompi/mca/mtl/ofi/mtl_ofi_component.c b/ompi/mca/mtl/ofi/mtl_ofi_component.c index d33f5d9a32e..949b78a7077 100644 --- a/ompi/mca/mtl/ofi/mtl_ofi_component.c +++ b/ompi/mca/mtl/ofi/mtl_ofi_component.c @@ -881,9 +881,10 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads, } /** - * Save the maximum inject size. + * Save the maximum sizes. */ ompi_mtl_ofi.max_inject_size = prov->tx_attr->inject_size; + ompi_mtl_ofi.max_msg_size = prov->ep_attr->max_msg_size; /** * The user is not allowed to exceed MTL_OFI_MAX_PROG_EVENT_COUNT. diff --git a/ompi/mca/mtl/ofi/mtl_ofi_types.h b/ompi/mca/mtl/ofi/mtl_ofi_types.h index 8b9fbe08a28..de9341fc51b 100644 --- a/ompi/mca/mtl/ofi/mtl_ofi_types.h +++ b/ompi/mca/mtl/ofi/mtl_ofi_types.h @@ -70,6 +70,9 @@ typedef struct mca_mtl_ofi_module_t { /** Maximum inject size */ size_t max_inject_size; + /** Largest message that can be sent in a single send. */ + size_t max_msg_size; + /** Maximum number of CQ events to read in OFI Progress */ int ofi_progress_event_count;