fuse: fix deadlock caused by wrong locking order

biger410 · Brian Maly · commit 01e2cf335371 · 2018-05-11T13:53:34.000-04:00
Orabug: 27719848 The locking order in fuse should be nn->fc->lock then nn->lock, mis-order locking will cause deadlock. The following deadlock was caused. PID 378084 asked lock in wrong order. PID: 378084 TASK: ffff8825421942c0 CPU: 2 COMMAND: "dbfs_client" #0 [ffff88207f846e70] crash_nmi_callback at ffffffff810326c6 #1 [ffff88207f846e80] notifier_call_chain at ffffffff81513115 #2 [ffff88207f846ec0] atomic_notifier_call_chain at ffffffff8151317a #3 [ffff88207f846ed0] notify_die at ffffffff815131ae #4 [ffff88207f846f00] default_do_nmi at ffffffff815106b9 #5 [ffff88207f846f30] do_nmi at ffffffff81510840 #6 [ffff88207f846f50] nmi at ffffffff8150fc10 [exception RIP: __ticket_spin_lock+25] RIP: ffffffff81040fe9 RSP: ffff8801f6d3b8e8 RFLAGS: 00000297 RAX: 00000000000068f8 RBX: 0000000000021000 RCX: ffff881fbd8e2d50 RDX: 00000000000068f7 RSI: ffff8801f6d3ba78 RDI: ffff883127828000 RBP: ffff8801f6d3b8e8 R8: ffff8801f6d3ba20 R9: 0000000000000001 R10: 0000000000000001 R11: 0000000000000001 R12: ffff883127828000 R13: ffff8801f6d3ba78 R14: ffff881fbd8e2cc4 R15: ffff881fbd8e2cc0 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- <NMI exception stack> --- #7 [ffff8801f6d3b8e8] __ticket_spin_lock at ffffffff81040fe9 #8 [ffff8801f6d3b8f0] _raw_spin_lock at ffffffff8150f16e #9 [ffff8801f6d3b900] fuse_get_unique at ffffffffa00fe2ce [fuse] #10 [ffff8801f6d3b920] fuse_read_batch_forget at ffffffffa00fe820 [fuse] #11 [ffff8801f6d3b9a0] fuse_dev_do_read at ffffffffa010052c [fuse] #12 [ffff8801f6d3ba70] fuse_dev_read at ffffffffa0100984 [fuse] #13 [ffff8801f6d3baf0] do_sync_read at ffffffff8116da52 #14 [ffff8801f6d3bc00] vfs_read at ffffffff8116e195 #15 [ffff8801f6d3bc30] sys_read at ffffffff8116e361 #16 [ffff8801f6d3bc80] _read_orig at ffffffffa05f411d [krg_10_5_0_3021_impOEL6-UEK4-smp-x86_64] #17 [ffff8801f6d3bce0] syscall_wrappers_generic_flow_with_param at ffffffffa05f0cc6 [krg_10_5_0_3021_impOEL6-UEK4-smp-x86_64] #18 [ffff8801f6d3bdb0] syscall_wrappers_generic_read.clone.2 at ffffffffa05f136b [krg_10_5_0_3021_impOEL6-UEK4-smp-x86_64] #19 [ffff8801f6d3bee0] SYS_read_common_wrap at ffffffffa05f6085 [krg_10_5_0_3021_impOEL6-UEK4-smp-x86_64] #20 [ffff8801f6d3bf70] SYS_read_wrap64 at ffffffffa05f617e [krg_10_5_0_3021_impOEL6-UEK4-smp-x86_64] #21 [ffff8801f6d3bf80] system_call_fastpath at ffffffff81517622 RIP: 00007f1492a3282d RSP: 00007f148a5f1448 RFLAGS: 00010206 RAX: 0000000000000000 RBX: ffffffff81517622 RCX: 00007f12de0cafd0 RDX: 0000000000021000 RSI: 00007f11e3938550 RDI: 0000000000000004 RBP: 00000000023f1110 R8: 00007ffce2baab50 R9: 000000000005c4e4 R10: 0000000000000024 R11: 0000000000000293 R12: ffffffffa05f617e R13: ffff8801f6d3bf78 R14: 00007f148a5f1e58 R15: 0000000000021000 ORIG_RAX: 0000000000000000 CS: 0033 SS: 002b PID: 38445 TASK: ffff881072a1c600 CPU: 19 COMMAND: "ggcmd" #0 [ffff88407f026e70] crash_nmi_callback at ffffffff810326c6 #1 [ffff88407f026e80] notifier_call_chain at ffffffff81513115 #2 [ffff88407f026ec0] atomic_notifier_call_chain at ffffffff8151317a #3 [ffff88407f026ed0] notify_die at ffffffff815131ae #4 [ffff88407f026f00] default_do_nmi at ffffffff815106b9 #5 [ffff88407f026f30] do_nmi at ffffffff81510840 #6 [ffff88407f026f50] nmi at ffffffff8150fc10 [exception RIP: __ticket_spin_lock+28] RIP: ffffffff81040fec RSP: ffff881070b8fb48 RFLAGS: 00000297 RAX: 000000000000a41c RBX: ffff881fbd8e2cc4 RCX: 0000000000051000 RDX: 000000000000a41b RSI: ffff8811edefac50 RDI: ffff881fbd8e2cc4 RBP: ffff881070b8fb48 R8: ffff8811edefac58 R9: 0000000000000003 R10: ffff88407ffd8e00 R11: 000000000000007d R12: ffff881fbd8e2cc0 R13: ffff8811edefac50 R14: ffff8811edefac58 R15: ffff8811edefac50 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- <NMI exception stack> --- #7 [ffff881070b8fb48] __ticket_spin_lock at ffffffff81040fec #8 [ffff881070b8fb50] _raw_spin_lock at ffffffff8150f16e #9 [ffff881070b8fb60] fuse_request_send_background_locked at ffffffffa00ffa97 [fuse] #10 [ffff881070b8fb90] fuse_send_writepage at ffffffffa0108301 [fuse] #11 [ffff881070b8fbc0] fuse_flush_writepages at ffffffffa01083f3 [fuse] #12 [ffff881070b8fc00] fuse_writepage_locked at ffffffffa0108683 [fuse] #13 [ffff881070b8fc60] fuse_writepage at ffffffffa010875e [fuse] #14 [ffff881070b8fc80] __writepage at ffffffff8111a8a7 #15 [ffff881070b8fca0] write_cache_pages at ffffffff8111bc06 #16 [ffff881070b8fdd0] generic_writepages at ffffffff8111bf31 #17 [ffff881070b8fe30] do_writepages at ffffffff8111bf95 #18 [ffff881070b8fe40] __filemap_fdatawrite_range at ffffffff8111166b #19 [ffff881070b8fe90] filemap_fdatawrite at ffffffff8111193f #20 [ffff881070b8fea0] filemap_write_and_wait at ffffffff81111985 #21 [ffff881070b8fec0] fuse_vma_close at ffffffffa010662c [fuse] #22 [ffff881070b8fed0] remove_vma at ffffffff8113c8b3 #23 [ffff881070b8fef0] do_munmap at ffffffff8113e8cf #24 [ffff881070b8ff50] sys_munmap at ffffffff8113e9e6 #25 [ffff881070b8ff80] system_call_fastpath at ffffffff81517622 RIP: 00007f3ed5cc84b7 RSP: 00007f3ed5100950 RFLAGS: 00000216 RAX: 000000000000000b RBX: ffffffff81517622 RCX: 0000000000140070 RDX: 0000000000000000 RSI: 00000000002fe000 RDI: 00007f3ed4abc000 RBP: 00007f3ed4abc1d8 R8: 00000000ffffffff R9: ffffffffffffc4f9 R10: 00000000000ce02f R11: 0000000000000246 R12: 00007f3ed4abc000 R13: 0000000000000000 R14: 00007f3ecc20d950 R15: 00007f3ecc007620 ORIG_RAX: 000000000000000b CS: 0033 SS: 002b OFF-MAINLINE/UEK5: nn->lock was introduced by oracle special fuse numa aware patches. OFF-UEK4: New lock fc->seq_lock was introduced, fc->lock not used in fuse_get_unique(). Signed-off-by: Junxiao Bi <junxiao.bi@oracle.com> Reviewed-by: Ashish Samant <ashish.samant@oracle.com> Signed-off-by: Brian Maly <brian.maly@oracle.com>
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
@@ -1096,7 +1096,6 @@ __releases(nn->lock)
 	struct fuse_in_header ih = {
 		.opcode = FUSE_FORGET,
 		.nodeid = forget->forget_one.nodeid,
-		.unique = fuse_get_unique(nn->fc),
 		.len = sizeof(ih) + sizeof(arg),
 	};
 
@@ -1105,6 +1104,7 @@ __releases(nn->lock)
 	if (nbytes < ih.len)
 		return -EINVAL;
 
+	ih.unique = fuse_get_unique(nn->fc);
 	err = fuse_copy_one(cs, &ih, sizeof(ih));
 	if (!err)
 		err = fuse_copy_one(cs, &arg, sizeof(arg));
@@ -1127,7 +1127,6 @@ __releases(nn->lock)
 	struct fuse_batch_forget_in arg = { .count = 0 };
 	struct fuse_in_header ih = {
 		.opcode = FUSE_BATCH_FORGET,
-		.unique = fuse_get_unique(nn->fc),
 		.len = sizeof(ih) + sizeof(arg),
 	};
 
@@ -1141,6 +1140,7 @@ __releases(nn->lock)
 	spin_unlock(&nn->lock);
 
 	arg.count = count;
+	ih.unique = fuse_get_unique(nn->fc);
 	ih.len += count * sizeof(struct fuse_forget_one);
 	err = fuse_copy_one(cs, &ih, sizeof(ih));
 	if (!err)
