Skip to content

btl/ofi: Use common provider include/exclude list #7975

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 10, 2020
Merged

btl/ofi: Use common provider include/exclude list #7975

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 4 additions & 24 deletions ompi/mca/mtl/ofi/mtl_ofi_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,26 +306,6 @@ ompi_mtl_ofi_progress_no_inline(void)
return ompi_mtl_ofi_progress();
}

static int
is_in_list(char **list, char *item)
{
int i = 0;

if ((NULL == list) || (NULL == item)) {
return 0;
}

while (NULL != list[i]) {
if (0 == strncasecmp(item, list[i], strlen(list[i]))) {
return 1;
} else {
i++;
}
}

return 0;
}

static struct fi_info*
select_ofi_provider(struct fi_info *providers,
char **include_list, char **exclude_list)
Expand All @@ -334,7 +314,7 @@ select_ofi_provider(struct fi_info *providers,

if (NULL != include_list) {
while ((NULL != prov) &&
(!is_in_list(include_list, prov->fabric_attr->prov_name))) {
(!opal_common_ofi_is_in_list(include_list, prov->fabric_attr->prov_name))) {
opal_output_verbose(1, opal_common_ofi.output,
"%s:%d: mtl:ofi: \"%s\" not in include list\n",
__FILE__, __LINE__,
Expand All @@ -343,7 +323,7 @@ select_ofi_provider(struct fi_info *providers,
}
} else if (NULL != exclude_list) {
while ((NULL != prov) &&
(is_in_list(exclude_list, prov->fabric_attr->prov_name))) {
(opal_common_ofi_is_in_list(exclude_list, prov->fabric_attr->prov_name))) {
opal_output_verbose(1, opal_common_ofi.output,
"%s:%d: mtl:ofi: \"%s\" in exclude list\n",
__FILE__, __LINE__,
Expand Down Expand Up @@ -733,8 +713,8 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
* this logic if the user specifies an include list without EFA or adds EFA
* to the exclude list.
*/
if ((include_list && is_in_list(include_list, "efa")) ||
(exclude_list && !is_in_list(exclude_list, "efa"))) {
if ((include_list && opal_common_ofi_is_in_list(include_list, "efa")) ||
(exclude_list && !opal_common_ofi_is_in_list(exclude_list, "efa"))) {
hints_dup = fi_dupinfo(hints);
hints_dup->caps &= ~(FI_LOCAL_COMM | FI_REMOTE_COMM);
hints_dup->fabric_attr->prov_name = strdup("efa");
Expand Down
33 changes: 28 additions & 5 deletions opal/mca/btl/ofi/btl_ofi_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,25 @@ static int mca_btl_ofi_init_device(struct fi_info *info);

/* validate information returned from fi_getinfo().
* return OPAL_ERROR if we dont have what we need. */
static int validate_info(struct fi_info *info, uint64_t required_caps)
static int validate_info(struct fi_info *info, uint64_t required_caps,
char **include_list, char **exclude_list)
{
int mr_mode;

if (NULL != include_list && !opal_common_ofi_is_in_list(include_list, info->fabric_attr->prov_name)) {
opal_output_verbose(1, opal_common_ofi.output,
"%s:%d: btl:ofi: \"%s\" not in include list\n",
__FILE__, __LINE__,
info->fabric_attr->prov_name);
return OPAL_ERROR;
} else if (NULL != exclude_list && opal_common_ofi_is_in_list(exclude_list, info->fabric_attr->prov_name)) {
opal_output_verbose(1, opal_common_ofi.output,
"%s:%d: btl:ofi: \"%s\" in exclude list\n",
__FILE__, __LINE__,
info->fabric_attr->prov_name);
return OPAL_ERROR;
}

BTL_VERBOSE(("validating device: %s", info->domain_attr->name));

/* we need exactly all the required bits */
Expand Down Expand Up @@ -219,7 +234,7 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
uint64_t progress_mode;
unsigned resource_count = 0;
struct mca_btl_base_module_t **base_modules;
char **include_list = NULL;
char **include_list = NULL, **exclude_list = NULL;

BTL_VERBOSE(("initializing ofi btl"));

Expand Down Expand Up @@ -264,10 +279,18 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
}

fabric_attr.prov_name = NULL;
/* Select the provider - sort of. we just take first element in list for now */

opal_output_verbose(1, opal_common_ofi.output,
"%s:%d: btl:ofi:provider_include = \"%s\"\n",
__FILE__, __LINE__, *opal_common_ofi.prov_include);
opal_output_verbose(1, opal_common_ofi.output,
"%s:%d: btl:ofi:provider_exclude = \"%s\"\n",
__FILE__, __LINE__, *opal_common_ofi.prov_exclude);

if (NULL != *opal_common_ofi.prov_include) {
include_list = opal_argv_split(*opal_common_ofi.prov_include, ',');
fabric_attr.prov_name = include_list[0];
} else if (NULL != *opal_common_ofi.prov_exclude) {
exclude_list = opal_argv_split(*opal_common_ofi.prov_exclude, ',');
}

domain_attr.mr_mode = MCA_BTL_OFI_REQUESTED_MR_MODE;
Expand Down Expand Up @@ -331,7 +354,7 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init (int *num_btl_modules,
info = info_list;

while(info) {
rc = validate_info(info, required_caps);
rc = validate_info(info, required_caps, include_list, exclude_list);
if (OPAL_SUCCESS == rc) {
/* Device passed sanity check, let's make a module.
*
Expand Down
19 changes: 19 additions & 0 deletions opal/mca/common/ofi/common_ofi.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,25 @@ OPAL_DECLSPEC opal_common_ofi_module_t opal_common_ofi = {

static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream";

OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item)
{
int i = 0;

if ((NULL == list) || (NULL == item)) {
return 0;
}

while (NULL != list[i]) {
if (0 == strncasecmp(item, list[i], strlen(list[i]))) {
return 1;
} else {
i++;
}
}

return 0;
}

OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_component_t *component)
{
static int registered = 0;
Expand Down
15 changes: 15 additions & 0 deletions opal/mca/common/ofi/common_ofi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,21 @@ OPAL_DECLSPEC void opal_common_ofi_mca_register(void);
OPAL_DECLSPEC void opal_common_ofi_mca_deregister(void);
OPAL_DECLSPEC struct fi_info* opal_common_ofi_select_ofi_provider(struct fi_info *providers,
char *framework_name);
/*
* @param list (IN) List of strings corresponding to lower providers.
* @param item (IN) Single string corresponding to a provider.
*
* @return 0 The lower provider of the item string is not in
* list or an input was NULL
* @return 1 The lower provider of the item string matches
* a string in the item list.
*
* This function will take a provider name string and a list of lower
* provider name strings as inputs. It will return true if the lower
* provider in the item string matches a lower provider in the list.
*
*/
OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item);
Copy link
Member

@bwbarrett bwbarrett Jul 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'd love some documentation :).

Copy link
Contributor Author

@wckzhang wckzhang Jul 31, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried my best, it's kind of a funky function. Maybe the provider list logic that currently exists should be changed to explicitly call out layered providers (ie. tcp, tcp;ofi_rxm) as separate providers.


END_C_DECLS

Expand Down