Skip to content

Commit 3deda3d

Browse files
zerothizerothi
committed
Added several new COMM_TYPE_<> splits
Using the underlying hardware identification to split communicators based on locality has been enabled using the MPI_Comm_Split_Type function. Currently implemented split's are: HWTHREAD CORE L1CACHE L2CACHE L3CACHE SOCKET NUMA NODE BOARD HOST CU CLUSTER However only NODE is defined in the standard which is why the remaning splits are referred to using the OMPI_ prefix instead of the standard MPI_ prefix. I have tested this using --without-hwloc and --with-hwloc=<path> which both give the same output. NOTE: I think something fishy is going on in the locality operators. In my test-program I couldn't get the correct split on these requests: NUMA, SOCKET, L3CACHE where I suspected a full communicator but only got one.
1 parent ffbf973 commit 3deda3d

File tree

5 files changed

+338
-18
lines changed

5 files changed

+338
-18
lines changed

‎ompi/communicator/comm.c‎

Lines changed: 252 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ ompi_comm_split_type(ompi_communicator_t *comm,
692692
int my_rsize;
693693
int mode;
694694
int rsize;
695-
int i, loc;
695+
int i, loc, found;
696696
int inter;
697697
int *results=NULL, *sorted=NULL;
698698
int *rresults=NULL, *rsorted=NULL;
@@ -711,7 +711,51 @@ ompi_comm_split_type(ompi_communicator_t *comm,
711711
/* --------------------------------------------------------- */
712712

713713
/* sort according to participation and rank. Gather information from everyone */
714-
myinfo[0] = (split_type == MPI_COMM_TYPE_SHARED) ? 1 : 0;
714+
/* allowed splitting types:
715+
CLUSTER
716+
CU
717+
HOST
718+
BOARD
719+
NODE
720+
NUMA
721+
SOCKET
722+
L3CACHE
723+
L2CACHE
724+
L1CACHE
725+
CORE
726+
HWTHREAD
727+
Even though HWTHREAD/CORE etc. is overkill they are here for consistency.
728+
They will most likely return a communicator which is equal to MPI_COMM_SELF
729+
Unless oversubscribing.
730+
*/
731+
myinfo[0] = 0; // default to no type splitting (also if non-recognized split-type)
732+
switch ( split_type ) {
733+
case OMPI_COMM_TYPE_HWTHREAD:
734+
myinfo[0] = 1; break;
735+
case OMPI_COMM_TYPE_CORE:
736+
myinfo[0] = 2; break;
737+
case OMPI_COMM_TYPE_L1CACHE:
738+
myinfo[0] = 3; break;
739+
case OMPI_COMM_TYPE_L2CACHE:
740+
myinfo[0] = 4; break;
741+
case OMPI_COMM_TYPE_L3CACHE:
742+
myinfo[0] = 5; break;
743+
case OMPI_COMM_TYPE_SOCKET:
744+
myinfo[0] = 6; break;
745+
case OMPI_COMM_TYPE_NUMA:
746+
myinfo[0] = 7; break;
747+
//case MPI_COMM_TYPE_SHARED: // the standard implemented type
748+
case OMPI_COMM_TYPE_NODE:
749+
myinfo[0] = 8; break;
750+
case OMPI_COMM_TYPE_BOARD:
751+
myinfo[0] = 9; break;
752+
case OMPI_COMM_TYPE_HOST:
753+
myinfo[0] = 10; break;
754+
case OMPI_COMM_TYPE_CU:
755+
myinfo[0] = 11; break;
756+
case OMPI_COMM_TYPE_CLUSTER:
757+
myinfo[0] = 12; break;
758+
}
715759
myinfo[1] = key;
716760

717761
size = ompi_comm_size ( comm );
@@ -731,13 +775,65 @@ ompi_comm_split_type(ompi_communicator_t *comm,
731775
if ( OMPI_SUCCESS != rc ) {
732776
goto exit;
733777
}
734-
778+
779+
/* check that all processors have been called with the same value */
780+
for ( i=0; i < size; i++) {
781+
if ( results[2*i] != myinfo[0] ) {
782+
rc = OMPI_ERR_BAD_PARAM;
783+
goto exit;
784+
}
785+
}
786+
735787
/* how many are participating and on my node? */
736788
for ( my_size = 0, i=0; i < size; i++) {
737-
if ( results[(2*i)+0] == 1) {
789+
if ( results[2*i] == 1 ) {
790+
if (OPAL_PROC_ON_LOCAL_HWTHREAD(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
791+
my_size++;
792+
}
793+
} else if ( results[2*i] == 2 ) {
794+
if (OPAL_PROC_ON_LOCAL_CORE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
795+
my_size++;
796+
}
797+
} else if ( results[2*i] == 3 ) {
798+
if (OPAL_PROC_ON_LOCAL_L1CACHE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
799+
my_size++;
800+
}
801+
} else if ( results[2*i] == 4 ) {
802+
if (OPAL_PROC_ON_LOCAL_L2CACHE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
803+
my_size++;
804+
}
805+
} else if ( results[2*i] == 5 ) {
806+
if (OPAL_PROC_ON_LOCAL_L3CACHE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
807+
my_size++;
808+
}
809+
} else if ( results[2*i] == 6 ) {
810+
if (OPAL_PROC_ON_LOCAL_SOCKET(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
811+
my_size++;
812+
}
813+
} else if ( results[2*i] == 7 ) {
814+
if (OPAL_PROC_ON_LOCAL_NUMA(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
815+
my_size++;
816+
}
817+
} else if ( results[2*i] == 8 ) {
738818
if (OPAL_PROC_ON_LOCAL_NODE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
739819
my_size++;
740820
}
821+
} else if ( results[2*i] == 9 ) {
822+
if (OPAL_PROC_ON_LOCAL_BOARD(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
823+
my_size++;
824+
}
825+
} else if ( results[2*i] == 10 ) {
826+
if (OPAL_PROC_ON_LOCAL_HOST(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
827+
my_size++;
828+
}
829+
} else if ( results[2*i] == 11 ) {
830+
if (OPAL_PROC_ON_LOCAL_CU(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
831+
my_size++;
832+
}
833+
} else if ( results[2*i] == 12 ) {
834+
if (OPAL_PROC_ON_LOCAL_CLUSTER(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
835+
my_size++;
836+
}
741837
}
742838
}
743839

@@ -755,13 +851,63 @@ ompi_comm_split_type(ompi_communicator_t *comm,
755851

756852
/* ok we can now fill this info */
757853
for( loc = 0, i = 0; i < size; i++ ) {
758-
if ( results[(2*i)+0] == 1) {
854+
found = 0;
855+
if ( results[2*i] == 1 ) {
856+
if (OPAL_PROC_ON_LOCAL_HWTHREAD(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
857+
found = 1;
858+
}
859+
} else if ( results[2*i] == 2 ) {
860+
if (OPAL_PROC_ON_LOCAL_CORE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
861+
found = 1;
862+
}
863+
} else if ( results[2*i] == 3 ) {
864+
if (OPAL_PROC_ON_LOCAL_L1CACHE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
865+
found = 1;
866+
}
867+
} else if ( results[2*i] == 4 ) {
868+
if (OPAL_PROC_ON_LOCAL_L2CACHE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
869+
found = 1;
870+
}
871+
} else if ( results[2*i] == 5 ) {
872+
if (OPAL_PROC_ON_LOCAL_L3CACHE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
873+
found = 1;
874+
}
875+
} else if ( results[2*i] == 6 ) {
876+
if (OPAL_PROC_ON_LOCAL_SOCKET(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
877+
found = 1;
878+
}
879+
} else if ( results[2*i] == 7 ) {
880+
if (OPAL_PROC_ON_LOCAL_NUMA(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
881+
found = 1;
882+
}
883+
} else if ( results[2*i] == 8 ) {
759884
if (OPAL_PROC_ON_LOCAL_NODE(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
760-
sorted[(2*loc)+0] = i; /* copy org rank */
761-
sorted[(2*loc)+1] = results[(2*i)+1]; /* copy key */
762-
loc++;
885+
found = 1;
886+
}
887+
} else if ( results[2*i] == 9 ) {
888+
if (OPAL_PROC_ON_LOCAL_BOARD(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
889+
found = 1;
890+
}
891+
} else if ( results[2*i] == 10 ) {
892+
if (OPAL_PROC_ON_LOCAL_HOST(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
893+
found = 1;
894+
}
895+
} else if ( results[2*i] == 11 ) {
896+
if (OPAL_PROC_ON_LOCAL_CU(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
897+
found = 1;
898+
}
899+
} else if ( results[2*i] == 12 ) {
900+
if (OPAL_PROC_ON_LOCAL_CLUSTER(ompi_group_peer_lookup(comm->c_local_group, i)->super.proc_flags)) {
901+
found = 1;
763902
}
764903
}
904+
905+
/* we have found and occupied the index (i) */
906+
if ( found == 1 ) {
907+
sorted[2*loc ] = i; /* copy org rank */
908+
sorted[2*loc+1] = results[2*i+1]; /* copy key */
909+
loc++;
910+
}
765911
}
766912

767913
/* the new array needs to be sorted so that it is in 'key' order */
@@ -800,10 +946,54 @@ ompi_comm_split_type(ompi_communicator_t *comm,
800946

801947
/* how many are participating and on my node? */
802948
for ( my_rsize = 0, i=0; i < rsize; i++) {
803-
if ( rresults[(2*i)+0] == 1) {
949+
if ( rresults[2*i] == 1 ) {
950+
if (OPAL_PROC_ON_LOCAL_HWTHREAD(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
951+
my_rsize++;
952+
}
953+
} else if ( rresults[2*i] == 2 ) {
954+
if (OPAL_PROC_ON_LOCAL_CORE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
955+
my_rsize++;
956+
}
957+
} else if ( rresults[2*i] == 3 ) {
958+
if (OPAL_PROC_ON_LOCAL_L1CACHE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
959+
my_rsize++;
960+
}
961+
} else if ( rresults[2*i] == 4 ) {
962+
if (OPAL_PROC_ON_LOCAL_L2CACHE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
963+
my_rsize++;
964+
}
965+
} else if ( rresults[2*i] == 5 ) {
966+
if (OPAL_PROC_ON_LOCAL_L3CACHE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
967+
my_rsize++;
968+
}
969+
} else if ( rresults[2*i] == 6 ) {
970+
if (OPAL_PROC_ON_LOCAL_SOCKET(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
971+
my_rsize++;
972+
}
973+
} else if ( rresults[2*i] == 7 ) {
974+
if (OPAL_PROC_ON_LOCAL_NUMA(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
975+
my_rsize++;
976+
}
977+
} else if ( rresults[2*i] == 8 ) {
804978
if (OPAL_PROC_ON_LOCAL_NODE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
805979
my_rsize++;
806980
}
981+
} else if ( rresults[2*i] == 9 ) {
982+
if (OPAL_PROC_ON_LOCAL_BOARD(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
983+
my_rsize++;
984+
}
985+
} else if ( rresults[2*i] == 10 ) {
986+
if (OPAL_PROC_ON_LOCAL_HOST(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
987+
my_rsize++;
988+
}
989+
} else if ( rresults[2*i] == 11 ) {
990+
if (OPAL_PROC_ON_LOCAL_CU(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
991+
my_rsize++;
992+
}
993+
} else if ( rresults[2*i] == 12 ) {
994+
if (OPAL_PROC_ON_LOCAL_CLUSTER(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
995+
my_rsize++;
996+
}
807997
}
808998
}
809999

@@ -816,12 +1006,61 @@ ompi_comm_split_type(ompi_communicator_t *comm,
8161006

8171007
/* ok we can now fill this info */
8181008
for( loc = 0, i = 0; i < rsize; i++ ) {
819-
if ( rresults[(2*i)+0] == 1) {
1009+
found = 0;
1010+
if ( rresults[2*i] == 1 ) {
1011+
if (OPAL_PROC_ON_LOCAL_HWTHREAD(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1012+
found = 1;
1013+
}
1014+
} else if ( rresults[2*i] == 2 ) {
1015+
if (OPAL_PROC_ON_LOCAL_CORE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1016+
found = 1;
1017+
}
1018+
} else if ( rresults[2*i] == 3 ) {
1019+
if (OPAL_PROC_ON_LOCAL_L1CACHE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1020+
found = 1;
1021+
}
1022+
} else if ( rresults[2*i] == 4 ) {
1023+
if (OPAL_PROC_ON_LOCAL_L2CACHE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1024+
found = 1;
1025+
}
1026+
} else if ( rresults[2*i] == 5 ) {
1027+
if (OPAL_PROC_ON_LOCAL_L3CACHE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1028+
found = 1;
1029+
}
1030+
} else if ( rresults[2*i] == 6 ) {
1031+
if (OPAL_PROC_ON_LOCAL_SOCKET(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1032+
found = 1;
1033+
}
1034+
} else if ( rresults[2*i] == 7 ) {
1035+
if (OPAL_PROC_ON_LOCAL_NUMA(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1036+
found = 1;
1037+
}
1038+
} else if ( rresults[2*i] == 8 ) {
8201039
if (OPAL_PROC_ON_LOCAL_NODE(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
821-
rsorted[(2*loc)+0] = i; /* org rank */
822-
rsorted[(2*loc)+1] = rresults[(2*i)+1]; /* key */
823-
loc++;
1040+
found = 1;
1041+
}
1042+
} else if ( rresults[2*i] == 9 ) {
1043+
if (OPAL_PROC_ON_LOCAL_BOARD(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1044+
found = 1;
1045+
}
1046+
} else if ( rresults[2*i] == 10 ) {
1047+
if (OPAL_PROC_ON_LOCAL_HOST(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1048+
found = 1;
8241049
}
1050+
} else if ( rresults[2*i] == 11 ) {
1051+
if (OPAL_PROC_ON_LOCAL_CU(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1052+
found = 1;
1053+
}
1054+
} else if ( rresults[2*i] == 12 ) {
1055+
if (OPAL_PROC_ON_LOCAL_CLUSTER(ompi_group_peer_lookup(comm->c_remote_group, i)->super.proc_flags)) {
1056+
found = 1;
1057+
}
1058+
}
1059+
1060+
if ( found == 1 ) {
1061+
rsorted[2*loc ] = i; /* org rank */
1062+
rsorted[2*loc+1] = rresults[2*i+1]; /* key */
1063+
loc++;
8251064
}
8261065
}
8271066

‎ompi/include/mpi.h.in‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,8 +667,20 @@ enum {
667667
* (see also mpif-common.h.fin).
668668
*/
669669
enum {
670-
MPI_COMM_TYPE_SHARED
670+
OMPI_COMM_TYPE_HWTHREAD,
671+
OMPI_COMM_TYPE_CORE,
672+
OMPI_COMM_TYPE_L1CACHE,
673+
OMPI_COMM_TYPE_L2CACHE,
674+
OMPI_COMM_TYPE_L3CACHE,
675+
OMPI_COMM_TYPE_SOCKET,
676+
OMPI_COMM_TYPE_NUMA,
677+
OMPI_COMM_TYPE_NODE,
678+
OMPI_COMM_TYPE_BOARD,
679+
OMPI_COMM_TYPE_HOST,
680+
OMPI_COMM_TYPE_CU,
681+
OMPI_COMM_TYPE_CLUSTER
671682
};
683+
#define MPI_COMM_TYPE_SHARED OMPI_COMM_TYPE_NODE
672684

673685
/*
674686
* MPIT Verbosity Levels

‎ompi/include/mpif-values.pl‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,19 @@ sub write_file {
354354
$constants->{MPI_COMBINER_RESIZED} = 17;
355355
$constants->{MPI_COMBINER_HINDEXED_BLOCK} = 18;
356356

357-
$constants->{MPI_COMM_TYPE_SHARED} = 0;
357+
$constants->{OMPI_COMM_TYPE_HWTHREAD} = 0;
358+
$constants->{OMPI_COMM_TYPE_CORE} = 1;
359+
$constants->{OMPI_COMM_TYPE_L1CACHE} = 2;
360+
$constants->{OMPI_COMM_TYPE_L2CACHE} = 3;
361+
$constants->{OMPI_COMM_TYPE_L3CACHE} = 4;
362+
$constants->{OMPI_COMM_TYPE_SOCKET} = 5;
363+
$constants->{OMPI_COMM_TYPE_NUMA} = 6;
364+
$constants->{OMPI_COMM_TYPE_NODE} = 7;
365+
$constants->{MPI_COMM_TYPE_SHARED} = 7;
366+
$constants->{OMPI_COMM_TYPE_BOARD} = 8;
367+
$constants->{OMPI_COMM_TYPE_HOST} = 9;
368+
$constants->{OMPI_COMM_TYPE_CU} = 10;
369+
$constants->{OMPI_COMM_TYPE_CLUSTER} = 11;
358370

359371
#----------------------------------------------------------------------------
360372

‎ompi/mpi/c/comm_split_type.c‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,19 @@ int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key,
6161
FUNC_NAME);
6262
}
6363

64-
if ( MPI_COMM_TYPE_SHARED != split_type &&
64+
if ( MPI_COMM_TYPE_SHARED != split_type && // Same as OMPI_COMM_TYPE_NODE
65+
OMPI_COMM_TYPE_CLUSTER != split_type &&
66+
OMPI_COMM_TYPE_CU != split_type &&
67+
OMPI_COMM_TYPE_HOST != split_type &&
68+
OMPI_COMM_TYPE_BOARD != split_type &&
69+
OMPI_COMM_TYPE_NODE != split_type && // Same as MPI_COMM_TYPE_SHARED
70+
OMPI_COMM_TYPE_NUMA != split_type &&
71+
OMPI_COMM_TYPE_SOCKET != split_type &&
72+
OMPI_COMM_TYPE_L3CACHE != split_type &&
73+
OMPI_COMM_TYPE_L2CACHE != split_type &&
74+
OMPI_COMM_TYPE_L1CACHE != split_type &&
75+
OMPI_COMM_TYPE_CORE != split_type &&
76+
OMPI_COMM_TYPE_HWTHREAD != split_type &&
6577
MPI_UNDEFINED != split_type ) {
6678
return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
6779
FUNC_NAME);

0 commit comments

Comments
 (0)