Skip to content

More scaling efficiencies #2775

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 21, 2017
Merged

More scaling efficiencies #2775

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
466cbd4
Rework the threading in oob/tcp so that daemons (including mpirun) us…
Jan 21, 2017
be3ef77
Improve packing efficiency by raising the initial buffer size and mod…
Jan 21, 2017
e8e5f81
Something not quite right about the revised allocation algos, so reve…
Jan 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions opal/dss/dss_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
Expand Down Expand Up @@ -47,12 +47,12 @@ BEGIN_C_DECLS
/*
* The default starting chunk size
*/
#define OPAL_DSS_DEFAULT_INITIAL_SIZE 128
#define OPAL_DSS_DEFAULT_INITIAL_SIZE 2048
/*
* The default threshold size when we switch from doubling the
* buffer size to addatively increasing it
*/
#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 1024
#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 4096

/*
* Internal type corresponding to size_t. Do not use this in
Expand Down
1 change: 1 addition & 0 deletions opal/dss/dss_internal_functions.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2017 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down
21 changes: 9 additions & 12 deletions orte/mca/odls/base/odls_base_default_fns.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
orte_proc_t *pptr, *dmn;
opal_buffer_t *bptr;
orte_app_context_t *app;
bool found;
orte_node_t *node;
bool newmap = false;

Expand Down Expand Up @@ -409,6 +408,13 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
if (NULL == jdata->map) {
jdata->map = OBJ_NEW(orte_job_map_t);
newmap = true;
} else if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED)) {
/* zero all the node map flags */
for (n=0; n < jdata->map->nodes->size; n++) {
if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, n))) {
ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
}
}
}

/* if we have a file map, then we need to load it */
Expand Down Expand Up @@ -454,17 +460,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
opal_pointer_array_add(dmn->node->procs, pptr);

/* add the node to the map, if not already there */
found = false;
for (k=0; k < jdata->map->nodes->size; k++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, k))) {
continue;
}
if (node->daemon == dmn) {
found = true;
break;
}
}
if (!found) {
if (!ORTE_FLAG_TEST(dmn->node, ORTE_NODE_FLAG_MAPPED)) {
OBJ_RETAIN(dmn->node);
opal_pointer_array_add(jdata->map->nodes, dmn->node);
if (newmap) {
Expand Down Expand Up @@ -497,6 +493,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx);
ORTE_FLAG_SET(app, ORTE_APP_FLAG_USED_ON_NODE);
}
ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED);
}

COMPLETE:
Expand Down
9 changes: 5 additions & 4 deletions orte/mca/oob/base/base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
* All rights reserved.
* Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2017 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand Down Expand Up @@ -54,13 +55,14 @@ OPAL_TIMING_DECLARE_EXT(ORTE_DECLSPEC, tm_oob)
* Convenience Typedef
*/
typedef struct {
opal_event_base_t *ev_base;
char *include;
char *exclude;
opal_list_t components;
opal_list_t actives;
int max_uri_length;
opal_hash_table_t peers;
bool use_module_threads;
int num_threads;
#if OPAL_ENABLE_TIMING
bool timing;
#endif
Expand Down Expand Up @@ -119,7 +121,7 @@ ORTE_DECLSPEC void orte_oob_base_send_nb(int fd, short args, void *cbdata);
__FILE__, __LINE__); \
cd = OBJ_NEW(orte_oob_send_t); \
cd->msg = (m); \
opal_event_set(orte_event_base, &cd->ev, -1, \
opal_event_set(orte_oob_base.ev_base, &cd->ev, -1, \
OPAL_EV_WRITE, \
orte_oob_base_send_nb, cd); \
opal_event_set_priority(&cd->ev, ORTE_MSG_PRI); \
Expand Down Expand Up @@ -173,7 +175,7 @@ OBJ_CLASS_DECLARATION(mca_oob_uri_req_t);
mca_oob_uri_req_t *rq; \
rq = OBJ_NEW(mca_oob_uri_req_t); \
rq->uri = strdup((u)); \
opal_event_set(orte_event_base, &(rq)->ev, -1, \
opal_event_set(orte_oob_base.ev_base, &(rq)->ev, -1, \
OPAL_EV_WRITE, \
orte_oob_base_set_addr, (rq)); \
opal_event_set_priority(&(rq)->ev, ORTE_MSG_PRI); \
Expand All @@ -193,4 +195,3 @@ ORTE_DECLSPEC void orte_oob_base_ft_event(int fd, short args, void *cbdata);

END_C_DECLS
#endif

24 changes: 19 additions & 5 deletions orte/mca/oob/base/oob_base_frame.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

#include "opal/class/opal_bitmap.h"
#include "orte/mca/mca.h"
#include "opal/runtime/opal_progress_threads.h"
#include "opal/util/output.h"
#include "opal/mca/base/base.h"

Expand All @@ -53,19 +54,20 @@
orte_oob_base_t orte_oob_base = {0};
OPAL_TIMING_DECLARE(tm_oob)


static int orte_oob_base_register(mca_base_register_flag_t flags)
{
if (ORTE_PROC_IS_APP || ORTE_PROC_IS_TOOL) {
orte_oob_base.use_module_threads = false;
orte_oob_base.num_threads = 0;
} else {
orte_oob_base.use_module_threads = true;
orte_oob_base.num_threads = 8;
}
(void)mca_base_var_register("orte", "oob", "base", "enable_module_progress_threads",
"Whether to independently progress OOB messages for each interface",
(void)mca_base_var_register("orte", "oob", "base", "num_progress_threads",
"Number of independent progress OOB messages for each interface",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_oob_base.use_module_threads);
&orte_oob_base.num_threads);

#if OPAL_ENABLE_TIMING
/* Detailed timing setup */
Expand Down Expand Up @@ -107,6 +109,11 @@ static int orte_oob_base_close(void)

OBJ_DESTRUCT(&orte_oob_base.peers);

if (ORTE_PROC_IS_APP || ORTE_PROC_IS_TOOL) {
opal_progress_thread_finalize(NULL);
} else {
opal_progress_thread_finalize("OOB-BASE");
}

OPAL_TIMING_EVENT((&tm_oob, "Finish"));
OPAL_TIMING_REPORT(orte_oob_base.timing, &tm_oob);
Expand All @@ -126,6 +133,13 @@ static int orte_oob_base_open(mca_base_open_flag_t flags)
opal_hash_table_init(&orte_oob_base.peers, 128);
OBJ_CONSTRUCT(&orte_oob_base.actives, opal_list_t);

if (ORTE_PROC_IS_APP || ORTE_PROC_IS_TOOL) {
orte_oob_base.ev_base = opal_progress_thread_init(NULL);
} else {
orte_oob_base.ev_base = opal_progress_thread_init("OOB-BASE");
}


#if OPAL_ENABLE_FT_CR == 1
/* register the FT events callback */
orte_state.add_job_state(ORTE_JOB_STATE_FT_CHECKPOINT, orte_oob_base_ft_event, ORTE_ERROR_PRI);
Expand Down
4 changes: 1 addition & 3 deletions orte/mca/oob/tcp/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2012-2013 Los Alamos National Security, LLC.
# All rights reserved
# Copyright (c) 2014 Intel, Inc. All rights reserved.
# Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
Expand All @@ -32,7 +32,6 @@ sources = \
oob_tcp_sendrecv.h \
oob_tcp_hdr.h \
oob_tcp_peer.h \
oob_tcp_ping.h \
oob_tcp.c \
oob_tcp_listener.c \
oob_tcp_common.c \
Expand All @@ -59,4 +58,3 @@ mca_oob_tcp_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(component_noinst)
libmca_oob_tcp_la_SOURCES = $(sources)
libmca_oob_tcp_la_LDFLAGS = -module -avoid-version

Loading