KVM: add spinlock optimization framework

Longpeng(Mike) · bonzini · commit 199b5763d329 · 2017-08-08T10:57:43.000+02:00
If a vcpu exits due to request a user mode spinlock, then
the spinlock-holder may be preempted in user mode or kernel mode.
(Note that not all architectures trap spin loops in user mode,
only AMD x86 and ARM/ARM64 currently do).

But if a vcpu exits in kernel mode, then the holder must be
preempted in kernel mode, so we should choose a vcpu in kernel mode
as a more likely candidate for the lock holder.

This introduces kvm_arch_vcpu_in_kernel() to decide whether the
vcpu is in kernel-mode when it's preempted.  kvm_vcpu_on_spin's
new argument says the same of the spinning VCPU.

Signed-off-by: Longpeng(Mike) &lt;longpeng2@huawei.com&gt;
Signed-off-by: Paolo Bonzini &lt;pbonzini@redhat.com&gt;
diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c
@@ -67,7 +67,7 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	if (kvm_vcpu_get_hsr(vcpu) & HSR_WFI_IS_WFE) {
 		trace_kvm_wfx(*vcpu_pc(vcpu), true);
 		vcpu->stat.wfe_exit_stat++;
-		kvm_vcpu_on_spin(vcpu);
+		kvm_vcpu_on_spin(vcpu, false);
 	} else {
 		trace_kvm_wfx(*vcpu_pc(vcpu), false);
 		vcpu->stat.wfi_exit_stat++;
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
@@ -84,7 +84,7 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
 		vcpu->stat.wfe_exit_stat++;
-		kvm_vcpu_on_spin(vcpu);
+		kvm_vcpu_on_spin(vcpu, false);
 	} else {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
 		vcpu->stat.wfi_exit_stat++;
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
@@ -98,6 +98,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 	return !!(vcpu->arch.pending_exceptions);
 }
 
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 {
 	return 1;
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
@@ -58,6 +58,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
 	return !!(v->arch.pending_exceptions) || kvm_request_pending(v);
 }
 
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 {
 	return 1;
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
@@ -150,7 +150,7 @@ static int __diag_time_slice_end(struct kvm_vcpu *vcpu)
 {
 	VCPU_EVENT(vcpu, 5, "%s", "diag time slice end");
 	vcpu->stat.diagnose_44++;
-	kvm_vcpu_on_spin(vcpu);
+	kvm_vcpu_on_spin(vcpu, false);
 	return 0;
 }
 
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
@@ -2447,6 +2447,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 	return kvm_s390_vcpu_has_irq(vcpu, 0);
 }
 
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+
 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
 {
 	atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
@@ -1274,7 +1274,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
 
 	switch (code) {
 	case HVCALL_NOTIFY_LONG_SPIN_WAIT:
-		kvm_vcpu_on_spin(vcpu);
+		kvm_vcpu_on_spin(vcpu, false);
 		break;
 	case HVCALL_POST_MESSAGE:
 	case HVCALL_SIGNAL_EVENT:
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
@@ -3749,7 +3749,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm)
 
 static int pause_interception(struct vcpu_svm *svm)
 {
-	kvm_vcpu_on_spin(&(svm->vcpu));
+	kvm_vcpu_on_spin(&svm->vcpu, false);
 	return 1;
 }
 
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
@@ -6781,7 +6781,7 @@ static int handle_pause(struct kvm_vcpu *vcpu)
 	if (ple_gap)
 		grow_ple_window(vcpu);
 
-	kvm_vcpu_on_spin(vcpu);
+	kvm_vcpu_on_spin(vcpu, false);
 	return kvm_skip_emulated_instruction(vcpu);
 }
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
@@ -8432,6 +8432,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 	return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
 }
 
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
@@ -720,7 +720,7 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu);
 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 int kvm_vcpu_yield_to(struct kvm_vcpu *target);
-void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu);
+void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu, bool usermode_vcpu_not_eligible);
 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
 
@@ -800,6 +800,7 @@ int kvm_arch_hardware_setup(void);
 void kvm_arch_hardware_unsetup(void);
 void kvm_arch_check_processor_compat(void *rtn);
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu);
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
 
 #ifndef __KVM_HAVE_ARCH_VM_ALLOC
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
@@ -416,6 +416,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
 		&& !v->arch.power_off && !v->arch.pause);
 }
 
+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+
 /* Just ensure a guest exit from a particular CPU */
 static void exit_vm_noop(void *info)
 {
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
@@ -2317,7 +2317,7 @@ static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
 #endif
 }
 
-void kvm_vcpu_on_spin(struct kvm_vcpu *me)
+void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
 {
 	struct kvm *kvm = me->kvm;
 	struct kvm_vcpu *vcpu;
@@ -2348,6 +2348,8 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me)
 				continue;
 			if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
 				continue;
+			if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu))
+				continue;
 			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
 				continue;
 

Original file line number	Diff line number	Diff line change
`@@ -98,6 +98,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)`
`98`	`98`	`return !!(vcpu->arch.pending_exceptions);`
`99`	`99`	`}`
`100`	`100`
	`101`	`+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)`
	`102`	`+{`
	`103`	`+ return false;`
	`104`	`+}`
	`105`	`+`
`101`	`106`	`int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)`
`102`	`107`	`{`
`103`	`108`	`return 1;`
Original file line number	Diff line number	Diff line change
`@@ -58,6 +58,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)`
`58`	`58`	`return !!(v->arch.pending_exceptions) \|\| kvm_request_pending(v);`
`59`	`59`	`}`
`60`	`60`
	`61`	`+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)`
	`62`	`+{`
	`63`	`+ return false;`
	`64`	`+}`
	`65`	`+`
`61`	`66`	`int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)`
`62`	`67`	`{`
`63`	`68`	`return 1;`
Original file line number	Diff line number	Diff line change
`@@ -150,7 +150,7 @@ static int __diag_time_slice_end(struct kvm_vcpu *vcpu)`
`150`	`150`	`{`
`151`	`151`	`VCPU_EVENT(vcpu, 5, "%s", "diag time slice end");`
`152`	`152`	`vcpu->stat.diagnose_44++;`
`153`		`- kvm_vcpu_on_spin(vcpu);`
	`153`	`+ kvm_vcpu_on_spin(vcpu, false);`
`154`	`154`	`return 0;`
`155`	`155`	`}`
`156`	`156`
Original file line number	Diff line number	Diff line change
`@@ -2447,6 +2447,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)`
`2447`	`2447`	`return kvm_s390_vcpu_has_irq(vcpu, 0);`
`2448`	`2448`	`}`
`2449`	`2449`
	`2450`	`+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)`
	`2451`	`+{`
	`2452`	`+ return false;`
	`2453`	`+}`
	`2454`	`+`
`2450`	`2455`	`void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)`
`2451`	`2456`	`{`
`2452`	`2457`	`atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);`
Original file line number	Diff line number	Diff line change
`@@ -3749,7 +3749,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm)`
`3749`	`3749`
`3750`	`3750`	`static int pause_interception(struct vcpu_svm *svm)`
`3751`	`3751`	`{`
`3752`		`- kvm_vcpu_on_spin(&(svm->vcpu));`
	`3752`	`+ kvm_vcpu_on_spin(&svm->vcpu, false);`
`3753`	`3753`	`return 1;`
`3754`	`3754`	`}`
`3755`	`3755`
Original file line number	Diff line number	Diff line change
`@@ -6781,7 +6781,7 @@ static int handle_pause(struct kvm_vcpu *vcpu)`
`6781`	`6781`	`if (ple_gap)`
`6782`	`6782`	`grow_ple_window(vcpu);`
`6783`	`6783`
`6784`		`- kvm_vcpu_on_spin(vcpu);`
	`6784`	`+ kvm_vcpu_on_spin(vcpu, false);`
`6785`	`6785`	`return kvm_skip_emulated_instruction(vcpu);`
`6786`	`6786`	`}`
`6787`	`6787`
Original file line number	Diff line number	Diff line change
`@@ -8432,6 +8432,11 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)`
`8432`	`8432`	`return kvm_vcpu_running(vcpu) \|\| kvm_vcpu_has_events(vcpu);`
`8433`	`8433`	`}`
`8434`	`8434`
	`8435`	`+bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)`
	`8436`	`+{`
	`8437`	`+ return false;`
	`8438`	`+}`
	`8439`	`+`
`8435`	`8440`	`int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)`
`8436`	`8441`	`{`
`8437`	`8442`	`return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;`