Skip to content

Commit 9a50fda

Browse files
hjelmnhjelmn
committed
btl: change argument type of BTL receive callbacks
This commit updates the btl interface to change the parameters passed to receive callbacks. The interface used to pass the tag, a btl base descriptor, and the callback context. Most of the values in the btl base descriptor were unused and only helped simplify the callbacks from the self btl. All of the arguments have now been replaced with a single receive callback descriptor. This descriptor contains the incoming endpoint, data segment(s), tag, and callback context. All btls have been updated to use the new callback and the btl interface version has been bumped to v3.2.0. As part of this change the descriptor argument (and the segments contained within it) have been marked as const. The were treated as const before but this change could allow the compiler to make better optimization decisions and will enforce that the callback does not attempt to change the data in the descriptor. Signed-off-by: Nathan Hjelm <[email protected]>
1 parent dbc5675 commit 9a50fda

22 files changed

+274
-259
lines changed

‎ompi/mca/pml/ob1/pml_ob1.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -311,8 +311,8 @@ void mca_pml_ob1_process_pending_rdma(void);
311311
* Compute the total number of bytes on supplied descriptor
312312
*/
313313
static inline size_t
314-
mca_pml_ob1_compute_segment_length_base(mca_btl_base_segment_t *segments,
315-
size_t count, size_t hdrlen)
314+
mca_pml_ob1_compute_segment_length_base (const mca_btl_base_segment_t *segments,
315+
size_t count, size_t hdrlen)
316316
{
317317
size_t i, length = 0;
318318

@@ -323,7 +323,7 @@ mca_pml_ob1_compute_segment_length_base(mca_btl_base_segment_t *segments,
323323
}
324324

325325
static inline size_t
326-
mca_pml_ob1_compute_segment_length_remote (size_t seg_size, void *segments,
326+
mca_pml_ob1_compute_segment_length_remote (size_t seg_size, const void *segments,
327327
size_t count, ompi_proc_t *rem_proc)
328328
{
329329
mca_btl_base_segment_t *segment = (mca_btl_base_segment_t *) segments;

‎ompi/mca/pml/ob1/pml_ob1_recvfrag.c‎

Lines changed: 93 additions & 119 deletions
Large diffs are not rendered by default.

‎ompi/mca/pml/ob1/pml_ob1_recvfrag.h‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
1515
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
1616
* reserved.
17+
* Copyright (c) 2020 Google, LLC. All rights reserved.
1718
* $COPYRIGHT$
1819
*
1920
* Additional copyrights may follow
@@ -113,60 +114,49 @@ do { \
113114
* Callback from BTL on receipt of a recv_frag (match).
114115
*/
115116

116-
extern void mca_pml_ob1_recv_frag_callback_match( mca_btl_base_module_t *btl,
117-
mca_btl_base_tag_t tag,
118-
mca_btl_base_descriptor_t* descriptor,
119-
void* cbdata );
117+
extern void mca_pml_ob1_recv_frag_callback_match (mca_btl_base_module_t *btl,
118+
const mca_btl_base_receive_descriptor_t *descriptor);
120119

121120
/**
122121
* Callback from BTL on receipt of a recv_frag (rndv).
123122
*/
124123

125-
extern void mca_pml_ob1_recv_frag_callback_rndv( mca_btl_base_module_t *btl,
126-
mca_btl_base_tag_t tag,
127-
mca_btl_base_descriptor_t* descriptor,
128-
void* cbdata );
124+
extern void mca_pml_ob1_recv_frag_callback_rndv (mca_btl_base_module_t *btl,
125+
const mca_btl_base_receive_descriptor_t *descriptor);
129126
/**
130127
* Callback from BTL on receipt of a recv_frag (rget).
131128
*/
132129

133-
extern void mca_pml_ob1_recv_frag_callback_rget( mca_btl_base_module_t *btl,
134-
mca_btl_base_tag_t tag,
135-
mca_btl_base_descriptor_t* descriptor,
136-
void* cbdata );
130+
extern void mca_pml_ob1_recv_frag_callback_rget (mca_btl_base_module_t *btl,
131+
const mca_btl_base_receive_descriptor_t *descriptor);
137132

138133
/**
139134
* Callback from BTL on receipt of a recv_frag (ack).
140135
*/
141136

142-
extern void mca_pml_ob1_recv_frag_callback_ack( mca_btl_base_module_t *btl,
143-
mca_btl_base_tag_t tag,
144-
mca_btl_base_descriptor_t* descriptor,
145-
void* cbdata );
137+
extern void mca_pml_ob1_recv_frag_callback_ack (mca_btl_base_module_t *btl,
138+
const mca_btl_base_receive_descriptor_t *descriptor);
139+
146140
/**
147141
* Callback from BTL on receipt of a recv_frag (frag).
148142
*/
149143

150-
extern void mca_pml_ob1_recv_frag_callback_frag( mca_btl_base_module_t *btl,
151-
mca_btl_base_tag_t tag,
152-
mca_btl_base_descriptor_t* descriptor,
153-
void* cbdata );
144+
extern void mca_pml_ob1_recv_frag_callback_frag (mca_btl_base_module_t *btl,
145+
const mca_btl_base_receive_descriptor_t *descriptor);
146+
154147
/**
155148
* Callback from BTL on receipt of a recv_frag (put).
156149
*/
157150

158-
extern void mca_pml_ob1_recv_frag_callback_put( mca_btl_base_module_t *btl,
159-
mca_btl_base_tag_t tag,
160-
mca_btl_base_descriptor_t* descriptor,
161-
void* cbdata );
151+
extern void mca_pml_ob1_recv_frag_callback_put (mca_btl_base_module_t *btl,
152+
const mca_btl_base_receive_descriptor_t *descriptor);
153+
162154
/**
163155
* Callback from BTL on receipt of a recv_frag (fin).
164156
*/
165157

166-
extern void mca_pml_ob1_recv_frag_callback_fin( mca_btl_base_module_t *btl,
167-
mca_btl_base_tag_t tag,
168-
mca_btl_base_descriptor_t* descriptor,
169-
void* cbdata );
158+
extern void mca_pml_ob1_recv_frag_callback_fin (mca_btl_base_module_t *btl,
159+
const mca_btl_base_receive_descriptor_t *descriptor);
170160

171161
/**
172162
* Extract the next fragment from the cant_match ordered list. This fragment

‎ompi/mca/pml/ob1/pml_ob1_recvreq.c‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
* and Technology (RIST). All rights reserved.
2121
* Copyright (c) 2018 Sandia National Laboratories
2222
* All rights reserved.
23+
* Copyright (c) 2020 Google, LLC. All rights reserved.
2324
* $COPYRIGHT$
2425
*
2526
* Additional copyrights may follow
@@ -517,7 +518,7 @@ int mca_pml_ob1_recv_request_get_frag (mca_pml_ob1_rdma_frag_t *frag)
517518

518519
void mca_pml_ob1_recv_request_progress_frag( mca_pml_ob1_recv_request_t* recvreq,
519520
mca_btl_base_module_t* btl,
520-
mca_btl_base_segment_t* segments,
521+
const mca_btl_base_segment_t* segments,
521522
size_t num_segments )
522523
{
523524
size_t bytes_received, data_offset = 0;
@@ -652,7 +653,7 @@ void mca_pml_ob1_recv_request_frag_copy_finished( mca_btl_base_module_t* btl,
652653

653654
void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq,
654655
mca_btl_base_module_t* btl,
655-
mca_btl_base_segment_t* segments,
656+
const mca_btl_base_segment_t* segments,
656657
size_t num_segments )
657658
{
658659
mca_pml_ob1_rget_hdr_t* hdr = (mca_pml_ob1_rget_hdr_t*)segments->seg_addr.pval;
@@ -806,7 +807,7 @@ void mca_pml_ob1_recv_request_progress_rget( mca_pml_ob1_recv_request_t* recvreq
806807

807808
void mca_pml_ob1_recv_request_progress_rndv( mca_pml_ob1_recv_request_t* recvreq,
808809
mca_btl_base_module_t* btl,
809-
mca_btl_base_segment_t* segments,
810+
const mca_btl_base_segment_t* segments,
810811
size_t num_segments )
811812
{
812813
size_t bytes_received = 0;
@@ -876,7 +877,7 @@ void mca_pml_ob1_recv_request_progress_rndv( mca_pml_ob1_recv_request_t* recvreq
876877
*/
877878
void mca_pml_ob1_recv_request_progress_match( mca_pml_ob1_recv_request_t* recvreq,
878879
mca_btl_base_module_t* btl,
879-
mca_btl_base_segment_t* segments,
880+
const mca_btl_base_segment_t* segments,
880881
size_t num_segments )
881882
{
882883
size_t bytes_received, data_offset = 0;
@@ -932,7 +933,7 @@ void mca_pml_ob1_recv_request_progress_match( mca_pml_ob1_recv_request_t* recvre
932933

933934
void mca_pml_ob1_recv_request_matched_probe( mca_pml_ob1_recv_request_t* recvreq,
934935
mca_btl_base_module_t* btl,
935-
mca_btl_base_segment_t* segments,
936+
const mca_btl_base_segment_t* segments,
936937
size_t num_segments )
937938
{
938939
size_t bytes_packed = 0;

‎ompi/mca/pml/ob1/pml_ob1_recvreq.h‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
* reserved.
1616
* Copyright (c) 2014 Research Organization for Information Science
1717
* and Technology (RIST). All rights reserved.
18+
* Copyright (c) 2020 Google, LLC. All rights reserved.
1819
*
1920
* $COPYRIGHT$
2021
*
@@ -239,7 +240,7 @@ static inline void prepare_recv_req_converter(mca_pml_ob1_recv_request_t *req)
239240
recv_req_matched(request, hdr)
240241

241242
static inline void recv_req_matched(mca_pml_ob1_recv_request_t *req,
242-
mca_pml_ob1_match_hdr_t *hdr)
243+
const mca_pml_ob1_match_hdr_t *hdr)
243244
{
244245
req->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
245246
req->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
@@ -278,7 +279,7 @@ do {
278279
uint32_t iov_count = 0; \
279280
size_t max_data = bytes_received; \
280281
size_t n, offset = seg_offset; \
281-
mca_btl_base_segment_t* segment = segments; \
282+
const mca_btl_base_segment_t *segment = segments; \
282283
\
283284
for( n = 0; n < num_segments; n++, segment++ ) { \
284285
if(offset >= segment->seg_len) { \
@@ -314,7 +315,7 @@ do {
314315
void mca_pml_ob1_recv_request_progress_match(
315316
mca_pml_ob1_recv_request_t* req,
316317
struct mca_btl_base_module_t* btl,
317-
mca_btl_base_segment_t* segments,
318+
const mca_btl_base_segment_t* segments,
318319
size_t num_segments);
319320

320321
/**
@@ -324,7 +325,7 @@ void mca_pml_ob1_recv_request_progress_match(
324325
void mca_pml_ob1_recv_request_progress_frag(
325326
mca_pml_ob1_recv_request_t* req,
326327
struct mca_btl_base_module_t* btl,
327-
mca_btl_base_segment_t* segments,
328+
const mca_btl_base_segment_t* segments,
328329
size_t num_segments);
329330

330331
#if OPAL_CUDA_SUPPORT
@@ -347,7 +348,7 @@ void mca_pml_ob1_recv_request_frag_copy_finished(struct mca_btl_base_module_t* b
347348
void mca_pml_ob1_recv_request_progress_rndv(
348349
mca_pml_ob1_recv_request_t* req,
349350
struct mca_btl_base_module_t* btl,
350-
mca_btl_base_segment_t* segments,
351+
const mca_btl_base_segment_t* segments,
351352
size_t num_segments);
352353

353354
/**
@@ -357,7 +358,7 @@ void mca_pml_ob1_recv_request_progress_rndv(
357358
void mca_pml_ob1_recv_request_progress_rget(
358359
mca_pml_ob1_recv_request_t* req,
359360
struct mca_btl_base_module_t* btl,
360-
mca_btl_base_segment_t* segments,
361+
const mca_btl_base_segment_t* segments,
361362
size_t num_segments);
362363

363364
/**
@@ -367,7 +368,7 @@ void mca_pml_ob1_recv_request_progress_rget(
367368
void mca_pml_ob1_recv_request_matched_probe(
368369
mca_pml_ob1_recv_request_t* req,
369370
struct mca_btl_base_module_t* btl,
370-
mca_btl_base_segment_t* segments,
371+
const mca_btl_base_segment_t* segments,
371372
size_t num_segments);
372373

373374
/**

‎ompi/mca/pml/ob1/pml_ob1_sendreq.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1216,9 +1216,9 @@ int mca_pml_ob1_send_request_put_frag( mca_pml_ob1_rdma_frag_t *frag )
12161216
* (3) Queue the RDMA put
12171217
*/
12181218

1219-
void mca_pml_ob1_send_request_put( mca_pml_ob1_send_request_t* sendreq,
1220-
mca_btl_base_module_t* btl,
1221-
mca_pml_ob1_rdma_hdr_t* hdr )
1219+
void mca_pml_ob1_send_request_put (mca_pml_ob1_send_request_t *sendreq,
1220+
mca_btl_base_module_t *btl,
1221+
const mca_pml_ob1_rdma_hdr_t *hdr)
12221222
{
12231223
mca_bml_base_endpoint_t *bml_endpoint = sendreq->req_endpoint;
12241224
mca_pml_ob1_rdma_frag_t* frag;

‎ompi/mca/pml/ob1/pml_ob1_sendreq.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -512,9 +512,9 @@ mca_pml_ob1_send_request_start( mca_pml_ob1_send_request_t* sendreq )
512512
* Initiate a put scheduled by the receiver.
513513
*/
514514

515-
void mca_pml_ob1_send_request_put( mca_pml_ob1_send_request_t* sendreq,
516-
mca_btl_base_module_t* btl,
517-
mca_pml_ob1_rdma_hdr_t* hdr );
515+
void mca_pml_ob1_send_request_put (mca_pml_ob1_send_request_t *sendreq,
516+
mca_btl_base_module_t *btl,
517+
const mca_pml_ob1_rdma_hdr_t *hdr);
518518

519519
int mca_pml_ob1_send_request_put_frag(mca_pml_ob1_rdma_frag_t* frag);
520520

0 commit comments

Comments
 (0)