Merge pull request #2775 from rhc54/topic/oob3

More scaling efficiencies

Author: Ralph Castain

opal/dss/dss_internal.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
@@ -47,12 +47,12 @@ BEGIN_C_DECLS
 /*
  * The default starting chunk size
  */
-#define OPAL_DSS_DEFAULT_INITIAL_SIZE  128
+#define OPAL_DSS_DEFAULT_INITIAL_SIZE  2048
 /*
  * The default threshold size when we switch from doubling the
  * buffer size to addatively increasing it
  */
-#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 1024
+#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 4096
 
 /*
  * Internal type corresponding to size_t.  Do not use this in

opal/dss/dss_internal_functions.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow

orte/mca/odls/base/odls_base_default_fns.c

@@ -261,7 +261,6 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
     orte_proc_t *pptr, *dmn;
     opal_buffer_t *bptr;
     orte_app_context_t *app;
-    bool found;
     orte_node_t *node;
     bool newmap = false;
 
@@ -409,6 +408,13 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
     if (NULL == jdata->map) {
         jdata->map = OBJ_NEW(orte_job_map_t);
         newmap = true;
+    } else if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED)) {
+        /* zero all the node map flags */
+        for (n=0; n < jdata->map->nodes->size; n++) {
+            if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, n))) {
+                ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
+            }
+        }
     }
 
     /* if we have a file map, then we need to load it */
@@ -454,17 +460,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
         opal_pointer_array_add(dmn->node->procs, pptr);
 
         /* add the node to the map, if not already there */
-        found = false;
-        for (k=0; k < jdata->map->nodes->size; k++) {
-            if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, k))) {
-                continue;
-            }
-            if (node->daemon == dmn) {
-                found = true;
-                break;
-            }
-        }
-        if (!found) {
+        if (!ORTE_FLAG_TEST(dmn->node, ORTE_NODE_FLAG_MAPPED)) {
             OBJ_RETAIN(dmn->node);
             opal_pointer_array_add(jdata->map->nodes, dmn->node);
             if (newmap) {
@@ -497,6 +493,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
             app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx);
             ORTE_FLAG_SET(app, ORTE_APP_FLAG_USED_ON_NODE);
         }
+        ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED);
     }
 
  COMPLETE:

orte/mca/oob/base/base.h

@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,13 +55,14 @@ OPAL_TIMING_DECLARE_EXT(ORTE_DECLSPEC, tm_oob)
  * Convenience Typedef
  */
 typedef struct {
+    opal_event_base_t *ev_base;
     char *include;
     char *exclude;
     opal_list_t components;
     opal_list_t actives;
     int max_uri_length;
     opal_hash_table_t peers;
-    bool use_module_threads;
+    int num_threads;
 #if OPAL_ENABLE_TIMING
     bool timing;
 #endif
@@ -119,7 +121,7 @@ ORTE_DECLSPEC void orte_oob_base_send_nb(int fd, short args, void *cbdata);
                             __FILE__, __LINE__);                        \
         cd = OBJ_NEW(orte_oob_send_t);                                  \
         cd->msg = (m);                                                  \
-        opal_event_set(orte_event_base, &cd->ev, -1,                    \
+        opal_event_set(orte_oob_base.ev_base, &cd->ev, -1,              \
                        OPAL_EV_WRITE,                                   \
                        orte_oob_base_send_nb, cd);                      \
         opal_event_set_priority(&cd->ev, ORTE_MSG_PRI);                 \
@@ -173,7 +175,7 @@ OBJ_CLASS_DECLARATION(mca_oob_uri_req_t);
         mca_oob_uri_req_t *rq;                                  \
         rq = OBJ_NEW(mca_oob_uri_req_t);                        \
         rq->uri = strdup((u));                                  \
-        opal_event_set(orte_event_base, &(rq)->ev, -1,          \
+        opal_event_set(orte_oob_base.ev_base, &(rq)->ev, -1,    \
                        OPAL_EV_WRITE,                           \
                        orte_oob_base_set_addr, (rq));           \
         opal_event_set_priority(&(rq)->ev, ORTE_MSG_PRI);       \
@@ -193,4 +195,3 @@ ORTE_DECLSPEC void orte_oob_base_ft_event(int fd, short args, void *cbdata);
 
 END_C_DECLS
 #endif
-

orte/mca/oob/base/oob_base_frame.c

@@ -29,6 +29,7 @@
 
 #include "opal/class/opal_bitmap.h"
 #include "orte/mca/mca.h"
+#include "opal/runtime/opal_progress_threads.h"
 #include "opal/util/output.h"
 #include "opal/mca/base/base.h"
 
@@ -53,19 +54,20 @@
 orte_oob_base_t orte_oob_base = {0};
 OPAL_TIMING_DECLARE(tm_oob)
 
+
 static int orte_oob_base_register(mca_base_register_flag_t flags)
 {
     if (ORTE_PROC_IS_APP || ORTE_PROC_IS_TOOL) {
-        orte_oob_base.use_module_threads = false;
+        orte_oob_base.num_threads = 0;
     } else {
-        orte_oob_base.use_module_threads = true;
+        orte_oob_base.num_threads = 8;
     }
-    (void)mca_base_var_register("orte", "oob", "base", "enable_module_progress_threads",
-                                "Whether to independently progress OOB messages for each interface",
+    (void)mca_base_var_register("orte", "oob", "base", "num_progress_threads",
+                                "Number of independent progress OOB messages for each interface",
                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
                                 OPAL_INFO_LVL_9,
                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                &orte_oob_base.use_module_threads);
+                                &orte_oob_base.num_threads);
 
 #if OPAL_ENABLE_TIMING
     /* Detailed timing setup */
@@ -107,6 +109,11 @@ static int orte_oob_base_close(void)
 
     OBJ_DESTRUCT(&orte_oob_base.peers);
 
+    if (ORTE_PROC_IS_APP || ORTE_PROC_IS_TOOL) {
+        opal_progress_thread_finalize(NULL);
+    } else {
+        opal_progress_thread_finalize("OOB-BASE");
+    }
 
     OPAL_TIMING_EVENT((&tm_oob, "Finish"));
     OPAL_TIMING_REPORT(orte_oob_base.timing, &tm_oob);
@@ -126,6 +133,13 @@ static int orte_oob_base_open(mca_base_open_flag_t flags)
     opal_hash_table_init(&orte_oob_base.peers, 128);
     OBJ_CONSTRUCT(&orte_oob_base.actives, opal_list_t);
 
+    if (ORTE_PROC_IS_APP || ORTE_PROC_IS_TOOL) {
+        orte_oob_base.ev_base = opal_progress_thread_init(NULL);
+    } else {
+        orte_oob_base.ev_base = opal_progress_thread_init("OOB-BASE");
+    }
+
+
 #if OPAL_ENABLE_FT_CR == 1
     /* register the FT events callback */
     orte_state.add_job_state(ORTE_JOB_STATE_FT_CHECKPOINT, orte_oob_base_ft_event, ORTE_ERROR_PRI);

orte/mca/oob/tcp/Makefile.am

@@ -12,7 +12,7 @@
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012-2013 Los Alamos National Security, LLC.
 #                         All rights reserved
-# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -32,7 +32,6 @@ sources = \
           oob_tcp_sendrecv.h \
           oob_tcp_hdr.h \
           oob_tcp_peer.h \
-          oob_tcp_ping.h \
           oob_tcp.c \
           oob_tcp_listener.c \
           oob_tcp_common.c \
@@ -59,4 +58,3 @@ mca_oob_tcp_la_LDFLAGS = -module -avoid-version
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_oob_tcp_la_SOURCES = $(sources)
 libmca_oob_tcp_la_LDFLAGS = -module -avoid-version
-