Enable the PMIx notification callback system and fix debugger attach

ompi/mca/rte/orte/rte_orte.h

@@ -91,7 +91,7 @@ OMPI_DECLSPEC void __opal_attribute_noreturn__
 #define OMPI_ERROR_LOG ORTE_ERROR_LOG
 
 /* Init and finalize objects and operations */
-#define ompi_rte_init(a, b) orte_init(a, b, ORTE_PROC_MPI)
+OMPI_DECLSPEC int ompi_rte_init(int *pargc, char ***pargv);
 #define ompi_rte_finalize() orte_finalize()
 OMPI_DECLSPEC void ompi_rte_wait_for_debugger(void);
 

ompi/mca/rte/orte/rte_orte_module.c

@@ -52,6 +52,79 @@
 
 extern ompi_rte_orte_component_t mca_rte_orte_component;
 
+typedef struct {
+    volatile bool active;
+    int status;
+    int errhandler;
+} errhandler_t;
+
+static void register_cbfunc(int status, int errhndler, void *cbdata)
+{
+    errhandler_t *cd = (errhandler_t*)cbdata;
+    cd->status = status;
+    cd->errhandler = errhndler;
+    cd->active = false;
+}
+
+static volatile bool wait_for_release = true;
+static int errhandler = -1;
+
+static void notify_cbfunc(int status,
+                          opal_list_t *procs,
+                          opal_list_t *info,
+                          opal_pmix_release_cbfunc_t cbfunc,
+                          void *cbdata)
+{
+    if (NULL != cbfunc) {
+        cbfunc(cbdata);
+    }
+    wait_for_release = false;
+}
+
+
+int ompi_rte_init(int *pargc, char ***pargv)
+{
+    int rc;
+    opal_list_t info;
+    opal_value_t val;
+    errhandler_t cd;
+
+    if (ORTE_SUCCESS != (rc = orte_init(pargc, pargv, ORTE_PROC_MPI))) {
+        return rc;
+    }
+
+    if (!orte_standalone_operation) {
+        /* register to receive any debugger release */
+        OBJ_CONSTRUCT(&info, opal_list_t);
+        OBJ_CONSTRUCT(&val, opal_value_t);
+        val.key = strdup(OPAL_PMIX_ERROR_NAME);
+        val.type = OPAL_INT;
+        val.data.integer = OPAL_ERR_DEBUGGER_RELEASE;
+        opal_list_append(&info, &val.super);
+        cd.status = ORTE_ERROR;
+        cd.errhandler = -1;
+        cd.active = true;
+
+        opal_pmix.register_errhandler(&info, notify_cbfunc, register_cbfunc, &cd);
+
+        /* let the MPI progress engine run while we wait for
+         * registration to complete */
+        OMPI_WAIT_FOR_COMPLETION(cd.active);
+        /* safely deconstruct the list */
+        opal_list_remove_first(&info);
+        OBJ_DESTRUCT(&val);
+        OBJ_DESTRUCT(&info);
+        if (OPAL_SUCCESS != cd.status) {
+            /* ouch - we are doomed */
+            ORTE_ERROR_LOG(cd.status);
+            return OMPI_ERROR;
+        }
+        errhandler = cd.errhandler;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 void ompi_rte_abort(int error_code, char *fmt, ...)
 {
     va_list arglist;
@@ -100,10 +173,10 @@ void ompi_rte_abort(int error_code, char *fmt, ...)
  * attaching debuggers -- see big comment in
  * orte/tools/orterun/debuggers.c explaining the two scenarios.
  */
+
 void ompi_rte_wait_for_debugger(void)
 {
     int debugger;
-    orte_rml_recv_cb_t xfer;
 
     /* See lengthy comment in orte/tools/orterun/debuggers.c about
        orte_in_parallel_debugger */
@@ -117,12 +190,12 @@ void ompi_rte_wait_for_debugger(void)
         /* if not, just return */
         return;
     }
-
     /* if we are being debugged, then we need to find
      * the correct plug-ins
      */
     ompi_debugger_setup_dlls();
 
+    /* wait for the debugger to attach */
     if (orte_standalone_operation) {
         /* spin until debugger attaches and releases us */
         while (MPIR_debug_gate == 0) {
@@ -133,23 +206,9 @@ void ompi_rte_wait_for_debugger(void)
 #endif
         }
     } else {
-        /* only the rank=0 proc waits for either a message from the
-         * HNP or for the debugger to attach - everyone else will just
-         * spin in * the grpcomm barrier in ompi_mpi_init until rank=0
-         * joins them.
-         */
-        if (0 != ORTE_PROC_MY_NAME->vpid) {
-            return;
-        }
-
-        /* VPID 0 waits for a message from the HNP */
-        OBJ_CONSTRUCT(&xfer, orte_rml_recv_cb_t);
-        xfer.active = true;
-        orte_rml.recv_buffer_nb(OMPI_NAME_WILDCARD,
-                                ORTE_RML_TAG_DEBUGGER_RELEASE,
-                                ORTE_RML_NON_PERSISTENT,
-                                orte_rml_recv_callback, &xfer);
-        /* let the MPI progress engine run while we wait */
-        OMPI_WAIT_FOR_COMPLETION(xfer.active);
+        /* now wait for the notification to occur */
+        OMPI_WAIT_FOR_COMPLETION(wait_for_release);
+        /* deregister the errhandler */
+        opal_pmix.deregister_errhandler(errhandler, NULL, NULL);
     }
 }

opal/dss/dss_compare.c

@@ -10,9 +10,9 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -449,3 +449,12 @@ int opal_dss_compare_jobid(opal_jobid_t *value1,
     return OPAL_EQUAL;
 }
 
+int opal_dss_compare_status(int *value1, int *value2, opal_data_type_t type)
+{
+    if (*value1 > *value2) return OPAL_VALUE1_GREATER;
+
+    if (*value2 > *value1) return OPAL_VALUE2_GREATER;
+
+    return OPAL_EQUAL;
+}
+

opal/dss/dss_copy.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -61,6 +61,7 @@ int opal_dss_std_copy(void **dest, void *src, opal_data_type_t type)
 
     case OPAL_INT:
     case OPAL_UINT:
+    case OPAL_STATUS:
         datasize = sizeof(int);
         break;
 

opal/dss/dss_internal.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
@@ -332,6 +332,9 @@ int opal_dss_pack_jobid(opal_buffer_t *buffer, const void *src,
 int opal_dss_pack_vpid(opal_buffer_t *buffer, const void *src,
                       int32_t num_vals, opal_data_type_t type);
 
+int opal_dss_pack_status(opal_buffer_t *buffer, const void *src,
+                         int32_t num_vals, opal_data_type_t type);
+
 /*
  * Internal unpack functions
  */
@@ -401,6 +404,8 @@ int opal_dss_unpack_jobid(opal_buffer_t *buffer, void *dest,
 int opal_dss_unpack_vpid(opal_buffer_t *buffer, void *dest,
                         int32_t *num_vals, opal_data_type_t type);
 
+int opal_dss_unpack_status(opal_buffer_t *buffer, void *dest,
+                           int32_t *num_vals, opal_data_type_t type);
 
 /*
  * Internal copy functions
@@ -497,6 +502,8 @@ int opal_dss_compare_jobid(opal_jobid_t *value1,
                            opal_jobid_t *value2,
                            opal_data_type_t type);
 
+int opal_dss_compare_status(int *value1, int *value2, opal_data_type_t type);
+
 /*
  * Internal print functions
  */
@@ -536,6 +543,7 @@ int opal_dss_print_time(char **output, char *prefix, time_t *src, opal_data_type
 int opal_dss_print_name(char **output, char *prefix, opal_process_name_t *name, opal_data_type_t type);
 int opal_dss_print_jobid(char **output, char *prefix, opal_process_name_t *src, opal_data_type_t type);
 int opal_dss_print_vpid(char **output, char *prefix, opal_process_name_t *src, opal_data_type_t type);
+int opal_dss_print_status(char **output, char *prefix, int *src, opal_data_type_t type);
 
 
 /*

opal/dss/dss_open_close.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -611,6 +611,17 @@ int opal_dss_open(void)
         return rc;
     }
 
+
+    tmp = OPAL_STATUS;
+    if (OPAL_SUCCESS != (rc = opal_dss.register_type(opal_dss_pack_status,
+                                          opal_dss_unpack_status,
+                                          (opal_dss_copy_fn_t)opal_dss_std_copy,
+                                          (opal_dss_compare_fn_t)opal_dss_compare_status,
+                                          (opal_dss_print_fn_t)opal_dss_print_status,
+                                          OPAL_DSS_UNSTRUCTURED,
+                                          "OPAL_STATUS", &tmp))) {
+        return rc;
+    }
     /* All done */
 
     opal_dss_initialized = true;

opal/dss/dss_pack.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2011-2013 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -1240,3 +1240,20 @@ int opal_dss_pack_vpid(opal_buffer_t *buffer, const void *src,
     return ret;
 }
 
+/*
+ * STATUS
+ */
+int opal_dss_pack_status(opal_buffer_t *buffer, const void *src,
+                         int32_t num_vals, opal_data_type_t type)
+{
+    int ret;
+
+    /* Turn around and pack the real type */
+    ret = opal_dss_pack_buffer(buffer, src, num_vals, OPAL_INT);
+    if (OPAL_SUCCESS != ret) {
+        OPAL_ERROR_LOG(ret);
+    }
+
+    return ret;
+}
+

opal/dss/dss_print.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -25,6 +25,7 @@
 #include "opal_stdint.h"
 #include <stdio.h>
 
+#include "opal/util/error.h"
 #include "opal/dss/dss_internal.h"
 
 int opal_dss_print(char **output, char *prefix, void *src, opal_data_type_t type)
@@ -1060,3 +1061,29 @@ int opal_dss_print_vpid(char **output, char *prefix,
 
     return OPAL_SUCCESS;
 }
+
+int opal_dss_print_status(char **output, char *prefix,
+                          int *src, opal_data_type_t type)
+{
+    char *prefx;
+
+    /* deal with NULL prefix */
+    if (NULL == prefix) asprintf(&prefx, " ");
+    else prefx = prefix;
+
+    /* if src is NULL, just print data type and return */
+    if (NULL == src) {
+        asprintf(output, "%sData type: OPAL_STATUS\tValue: NULL pointer", prefx);
+        if (prefx != prefix) {
+            free(prefx);
+        }
+        return OPAL_SUCCESS;
+    }
+
+    asprintf(output, "%sData type: OPAL_STATUS\tValue: %s", prefx, opal_strerror(*src));
+    if (prefx != prefix) {
+        free(prefx);
+    }
+
+    return OPAL_SUCCESS;
+}

opal/dss/dss_types.h

@@ -13,9 +13,9 @@
  * Copyright (c) 2007-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights
  *                         reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -120,6 +120,8 @@ typedef struct {
 #define    OPAL_NAME                (opal_data_type_t)   50
 #define    OPAL_JOBID               (opal_data_type_t)   51
 #define    OPAL_VPID                (opal_data_type_t)   52
+#define    OPAL_STATUS              (opal_data_type_t)   53
+
     /* OPAL Dynamic */
 #define    OPAL_DSS_ID_DYNAMIC      (opal_data_type_t)  100
 
@@ -245,6 +247,7 @@ typedef struct {
         float fval;
         double dval;
         struct timeval tv;
+        int status;
         opal_process_name_t name;
         opal_bool_array_t flag_array;
         opal_uint8_array_t byte_array;

opal/dss/dss_unpack.c

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012-2015 Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014      Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -1519,3 +1519,20 @@ int opal_dss_unpack_vpid(opal_buffer_t *buffer, void *dest,
 
     return ret;
 }
+
+/*
+ * STATUS
+ */
+int opal_dss_unpack_status(opal_buffer_t *buffer, void *dest,
+                           int32_t *num_vals, opal_data_type_t type)
+{
+    int ret;
+
+    /* Turn around and unpack the real type */
+    ret = opal_dss_unpack_buffer(buffer, dest, num_vals, OPAL_INT);
+    if (OPAL_SUCCESS != ret) {
+        OPAL_ERROR_LOG(ret);
+    }
+
+    return ret;
+}