1.Fix smp race between kernel vm86 BIOS calling and userland vm86 mode code,

  remove global variable in_vm86call, set vm86 calling flag in PCB flags.

2.Fix vm86 BIOS calling preempted problem by changing vm86_lock mutex type
  from MTX_DEF to MTX_SPIN. vm86pcb is not remembered in thread struct,
  when the thread calling vm86 BIOS is preempted by interrupt thread,
  and later switching back to the thread would cause incorrect context be
  loaded into CPU registers, this leads to kernel crash.

5 files changed, 14 insertions, 11 deletions

diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 8eeacbc..0ebb661 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -281,8 +281,9 @@ doreti_next:
 	 */
 	testl	$PSL_VM,TF_EFLAGS(%esp)	/* are we in vm86 mode? */
 	jz	doreti_notvm86
-	cmpl	$1,in_vm86call		/* are we in a vm86 call? XXXSMP */
-	jne	doreti_ast		/* can handle ASTs now if not */
+	movl	PCPU(CURPCB),%ecx
+	testl	$PCB_VM86CALL,PCB_FLAGS(%ecx)	/* are we in a vm86 call? */
+	jz	doreti_ast		/* can handle ASTS now if not */
   	jmp	doreti_exit
 
 doreti_notvm86:
diff --git a/sys/amd64/amd64/exception.s b/sys/amd64/amd64/exception.s
index 8eeacbc..0ebb661 100644
--- a/sys/amd64/amd64/exception.s
+++ b/sys/amd64/amd64/exception.s
@@ -281,8 +281,9 @@ doreti_next:
 	 */
 	testl	$PSL_VM,TF_EFLAGS(%esp)	/* are we in vm86 mode? */
 	jz	doreti_notvm86
-	cmpl	$1,in_vm86call		/* are we in a vm86 call? XXXSMP */
-	jne	doreti_ast		/* can handle ASTs now if not */
+	movl	PCPU(CURPCB),%ecx
+	testl	$PCB_VM86CALL,PCB_FLAGS(%ecx)	/* are we in a vm86 call? */
+	jz	doreti_ast		/* can handle ASTS now if not */
   	jmp	doreti_exit
 
 doreti_notvm86:
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 5a8cf70..4785645 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -146,6 +146,7 @@ ASSYM(PCB_SAVEFPU_SIZE, sizeof(union savefpu));
 ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
 
 ASSYM(PCB_SIZE, sizeof(struct pcb));
+ASSYM(PCB_VM86CALL, PCB_VM86CALL);
 
 ASSYM(TF_TRAPNO, offsetof(struct trapframe, tf_trapno));
 ASSYM(TF_ERR, offsetof(struct trapframe, tf_err));
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index a23ce49..54c4af8 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -260,7 +260,8 @@ trap(frame)
 #endif	/* DEVICE_POLLING */
 
         if ((ISPL(frame.tf_cs) == SEL_UPL) ||
-	    ((frame.tf_eflags & PSL_VM) && !in_vm86call)) {
+	    ((frame.tf_eflags & PSL_VM) && 
+		!(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL))) {
 		/* user trap */
 
 		sticks = td->td_kse->ke_sticks;
@@ -311,9 +312,7 @@ trap(frame)
 		case T_PROTFLT:		/* general protection fault */
 		case T_STKFLT:		/* stack fault */
 			if (frame.tf_eflags & PSL_VM) {
-				mtx_lock(&Giant);
 				i = vm86_emulate((struct vm86frame *)&frame);
-				mtx_unlock(&Giant);
 				if (i == 0)
 					goto user;
 				break;
@@ -466,9 +465,7 @@ trap(frame)
 		case T_PROTFLT:		/* general protection fault */
 		case T_STKFLT:		/* stack fault */
 			if (frame.tf_eflags & PSL_VM) {
-				mtx_lock(&Giant);
 				i = vm86_emulate((struct vm86frame *)&frame);
-				mtx_unlock(&Giant);
 				if (i != 0)
 					/*
 					 * returns to original process
@@ -482,7 +479,7 @@ trap(frame)
 			/* FALL THROUGH */
 
 		case T_SEGNPFLT:	/* segment not present fault */
-			if (in_vm86call)
+			if (PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)
 				break;
 
 			if (td->td_intr_nesting_level != 0)
@@ -584,7 +581,8 @@ trap(frame)
 			 * debugging the kernel.
 			 */
 			/* XXX Giant */
-			if (user_dbreg_trap() && !in_vm86call) {
+			if (user_dbreg_trap() && 
+			   !(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)) {
 				/*
 				 * Reset breakpoint bits because the
 				 * processor doesn't
diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h
index 013fc00..c7a837b 100644
--- a/sys/amd64/include/pcb.h
+++ b/sys/amd64/include/pcb.h
@@ -67,6 +67,8 @@ struct pcb {
 #define	PCB_DBREGS	0x02	/* process using debug registers */
 #define	PCB_NPXTRAP	0x04	/* npx trap pending */
 #define	PCB_NPXINITDONE	0x08	/* fpu state is initialized */
+#define	PCB_VM86CALL	0x10	/* in vm86 call */
+
 	caddr_t	pcb_onfault;	/* copyin/out fault recovery */
 	int	pcb_gs;
 	struct	pcb_ext	*pcb_ext;	/* optional pcb extension */