Skip to content

Commit 158f3e1

Browse files
markallemarkalle
committed
[from mpich] lock contiguous indpendent write when data sieving write is not disabled
Mpich PR pmodels/mpich#4995 has a commit titled lock contiguous indpendent write when data sieving write is not disabled This is a fix for the testcase https://gist.github.com/markalle/d7da240c19e57f095c5d1b13240dae24 % mpicc -o x romio_write_timing.c % mpirun -np 4 ./x Note: you need to use a filesystem that uses ADIOI_GEN_WriteStrided to hit the bug. I was using GPFS. Signed-off-by: Mark Allen <[email protected]>
1 parent 8bc0d98 commit 158f3e1

File tree

2 files changed

+38
-24
lines changed

2 files changed

+38
-24
lines changed

‎ompi/mca/io/romio321/romio/adio/ad_lustre/ad_lustre_wrstr.c‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
1919
ADIO_EXPLICIT_OFFSET, writebuf_off, \
2020
&status1, error_code); \
21-
if (!(fd->atomicity)) \
21+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
2222
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
2323
if (*error_code != MPI_SUCCESS) { \
2424
*error_code = MPIO_Err_create_code(*error_code, \
@@ -35,7 +35,7 @@
3535
writebuf_len = (unsigned) ADIOI_MIN(end_offset - writebuf_off + 1, \
3636
(writebuf_off / stripe_size + 1) * \
3737
stripe_size - writebuf_off); \
38-
if (!(fd->atomicity)) \
38+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
3939
ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
4040
ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, \
4141
ADIO_EXPLICIT_OFFSET, \
@@ -58,7 +58,7 @@
5858
while (write_sz != req_len) { \
5959
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
6060
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
61-
if (!(fd->atomicity)) \
61+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
6262
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
6363
if (*error_code != MPI_SUCCESS) { \
6464
*error_code = MPIO_Err_create_code(*error_code, \
@@ -75,7 +75,7 @@
7575
writebuf_len = (unsigned) ADIOI_MIN(end_offset - writebuf_off + 1, \
7676
(writebuf_off / stripe_size + 1) * \
7777
stripe_size - writebuf_off); \
78-
if (!(fd->atomicity)) \
78+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
7979
ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
8080
ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, \
8181
ADIO_EXPLICIT_OFFSET, \
@@ -221,8 +221,9 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
221221
writebuf_off = 0;
222222
writebuf_len = 0;
223223

224-
/* if atomicity is true, lock the region to be accessed */
225-
if (fd->atomicity)
224+
/* if atomicity is true or data sieving is not disable, lock the region
225+
* to be accessed */
226+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
226227
ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize);
227228

228229
for (j = 0; j < count; j++) {
@@ -241,7 +242,7 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
241242
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,
242243
error_code);
243244

244-
if (fd->atomicity)
245+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
245246
ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize);
246247
if (*error_code != MPI_SUCCESS) {
247248
ADIOI_Free(writebuf);
@@ -325,9 +326,13 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
325326
userbuf_off = 0;
326327
ADIOI_BUFFERED_WRITE_WITHOUT_READ
327328
/* write the buffer out finally */
329+
if (fd->hints->ds_write != ADIOI_HINT_DISABLE)
330+
ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
328331
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,
329332
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,
330333
error_code);
334+
if (fd->hints->ds_write != ADIOI_HINT_DISABLE)
335+
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
331336

332337
if (file_ptr_type == ADIO_INDIVIDUAL) {
333338
/* update MPI-IO file pointer to point to the first byte
@@ -378,8 +383,9 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
378383
fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
379384
}
380385

381-
/* if atomicity is true, lock the region to be accessed */
382-
if (fd->atomicity)
386+
/* if atomicity is true or data sieving is not disable, lock the region
387+
* to be accessed */
388+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
383389
ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
384390

385391
writebuf_off = 0;
@@ -502,11 +508,11 @@ void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, const void *buf, int count,
502508
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,
503509
ADIO_EXPLICIT_OFFSET,
504510
writebuf_off, &status1, error_code);
505-
if (!(fd->atomicity))
511+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE)
506512
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
507513
if (*error_code != MPI_SUCCESS) return;
508514
}
509-
if (fd->atomicity)
515+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
510516
ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
511517

512518
ADIOI_Free(writebuf);

‎ompi/mca/io/romio321/romio/adio/common/ad_write_str.c‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@
1414
if (writebuf_len) { \
1515
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
1616
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
17-
if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
17+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
18+
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
1819
if (*error_code != MPI_SUCCESS) { \
1920
*error_code = MPIO_Err_create_code(*error_code, \
2021
MPIR_ERR_RECOVERABLE, myname, \
@@ -25,7 +26,8 @@
2526
} \
2627
writebuf_off = req_off; \
2728
writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
28-
if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
29+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
30+
ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
2931
ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, \
3032
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
3133
if (*error_code != MPI_SUCCESS) { \
@@ -42,7 +44,8 @@
4244
while (write_sz != req_len) { \
4345
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
4446
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
45-
if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
47+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
48+
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
4649
if (*error_code != MPI_SUCCESS) { \
4750
*error_code = MPIO_Err_create_code(*error_code, \
4851
MPIR_ERR_RECOVERABLE, myname, \
@@ -54,7 +57,8 @@
5457
userbuf_off += write_sz; \
5558
writebuf_off += writebuf_len; \
5659
writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
57-
if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
60+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE) \
61+
ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
5862
ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, \
5963
ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
6064
if (*error_code != MPI_SUCCESS) { \
@@ -191,9 +195,10 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
191195
writebuf = (char *) ADIOI_Malloc(max_bufsize);
192196
writebuf_len = (unsigned) (ADIOI_MIN(max_bufsize, end_offset-writebuf_off+1));
193197

194-
/* if atomicity is true, lock the region to be accessed */
195-
if (fd->atomicity)
196-
ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
198+
/* if atomicity is true or data sieving is not disable, lock the region
199+
* to be accessed */
200+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
201+
ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset - start_off + 1);
197202

198203
for (j=0; j<count; j++)
199204
{
@@ -210,7 +215,7 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
210215
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
211216
writebuf_off, &status1, error_code);
212217

213-
if (fd->atomicity)
218+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
214219
ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
215220

216221
if (*error_code != MPI_SUCCESS) goto fn_exit;
@@ -287,8 +292,10 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
287292
* datatypes, instead of a count of bytes (which might overflow)
288293
* Other WriteContig calls in this path are operating on data
289294
* sieving buffer */
295+
ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, bufsize);
290296
ADIO_WriteContig(fd, buf, count, datatype, ADIO_EXPLICIT_OFFSET,
291297
offset, status, error_code);
298+
ADIOI_UNLOCK(fd, offset, SEEK_SET, bufsize);
292299

293300
if (file_ptr_type == ADIO_INDIVIDUAL) {
294301
/* update MPI-IO file pointer to point to the first byte
@@ -338,9 +345,10 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
338345
fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i_offset);
339346
}
340347

341-
/* if atomicity is true, lock the region to be accessed */
342-
if (fd->atomicity)
343-
ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
348+
/* if atomicity is true or data sieving is not disable, lock the region
349+
* to be accessed */
350+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
351+
ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset - start_off + 1);
344352

345353
writebuf_off = 0;
346354
writebuf_len = 0;
@@ -460,11 +468,11 @@ void ADIOI_GEN_WriteStrided(ADIO_File fd, const void *buf, int count,
460468
if (writebuf_len) {
461469
ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
462470
writebuf_off, &status1, error_code);
463-
if (!(fd->atomicity))
471+
if (!fd->atomicity && fd->hints->ds_write == ADIOI_HINT_DISABLE)
464472
ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
465473
if (*error_code != MPI_SUCCESS) goto fn_exit;
466474
}
467-
if (fd->atomicity)
475+
if (fd->atomicity || fd->hints->ds_write != ADIOI_HINT_DISABLE)
468476
ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
469477

470478
if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;

0 commit comments

Comments
 (0)