For the page fault handler, save %cr2 in the outer trap handler so that

we do not have to run so long with interrupts disabled.  This involved
creating tf_addr in the trapframe.  Reorganize the trap stubs so that
they consistently reserve the stack space and initialize any missing
bits.

Approved by:	re (amd64 stuff)

6 files changed, 76 insertions, 52 deletions

diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index e36bcb1..0469a9a 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -74,50 +74,68 @@
  */
 #define	IDTVEC(name)	ALIGN_TEXT; .globl __CONCAT(X,name); \
 			.type __CONCAT(X,name),@function; __CONCAT(X,name):
-#define	TRAP(a)		pushq $(a) ; jmp alltraps
-#define	TRAP_NOEN(a)	pushq $(a) ; jmp alltraps_noen
 
 MCOUNT_LABEL(user)
 MCOUNT_LABEL(btrap)
 
-IDTVEC(div)
-	pushq $0; TRAP(T_DIVIDE)
+/* Traps that we leave interrupts disabled for.. */
+#define	TRAP_NOEN(a)	\
+	subq $TF_RIP,%rsp; \
+	movq $(a),TF_TRAPNO(%rsp) ; \
+	movq $0,TF_ADDR(%rsp) ; \
+	movq $0,TF_ERR(%rsp) ; \
+	jmp alltraps_noen
 IDTVEC(dbg)
-	pushq $0; TRAP_NOEN(T_TRCTRAP)
-IDTVEC(nmi)
-	pushq $0; TRAP(T_NMI)
+	TRAP_NOEN(T_TRCTRAP)
 IDTVEC(bpt)
-	pushq $0; TRAP_NOEN(T_BPTFLT)
+	TRAP_NOEN(T_BPTFLT)
+
+/* Regular traps; The cpu does not supply tf_err for these. */
+#define	TRAP(a)	 \
+	subq $TF_RIP,%rsp; \
+	movq $(a),TF_TRAPNO(%rsp) ; \
+	movq $0,TF_ADDR(%rsp) ; \
+	movq $0,TF_ERR(%rsp) ; \
+	jmp alltraps
+IDTVEC(div)
+	TRAP(T_DIVIDE)
+IDTVEC(nmi)
+	TRAP(T_NMI)
 IDTVEC(ofl)
-	pushq $0; TRAP(T_OFLOW)
+	TRAP(T_OFLOW)
 IDTVEC(bnd)
-	pushq $0; TRAP(T_BOUND)
+	TRAP(T_BOUND)
 IDTVEC(ill)
-	pushq $0; TRAP(T_PRIVINFLT)
+	TRAP(T_PRIVINFLT)
 IDTVEC(dna)
-	pushq $0; TRAP(T_DNA)
+	TRAP(T_DNA)
 IDTVEC(fpusegm)
-	pushq $0; TRAP(T_FPOPFLT)
+	TRAP(T_FPOPFLT)
+IDTVEC(mchk)
+	TRAP(T_MCHK)
+IDTVEC(rsvd)
+	TRAP(T_RESERVED)
+IDTVEC(fpu)
+	TRAP(T_ARITHTRAP)
+IDTVEC(xmm)
+	TRAP(T_XMMFLT)
+
+/* This group of traps have tf_err already pushed by the cpu */
+#define	TRAP_ERR(a)	\
+	subq $TF_ERR,%rsp; \
+	movq $(a),TF_TRAPNO(%rsp) ; \
+	movq $0,TF_ADDR(%rsp) ; \
+	jmp alltraps_noen
 IDTVEC(tss)
-	TRAP(T_TSSFLT)
+	TRAP_ERR(T_TSSFLT)
 IDTVEC(missing)
-	TRAP(T_SEGNPFLT)
+	TRAP_ERR(T_SEGNPFLT)
 IDTVEC(stk)
-	TRAP(T_STKFLT)
+	TRAP_ERR(T_STKFLT)
 IDTVEC(prot)
-	TRAP(T_PROTFLT)
-IDTVEC(page)
-	TRAP_NOEN(T_PAGEFLT)
-IDTVEC(mchk)
-	pushq $0; TRAP(T_MCHK)
-IDTVEC(rsvd)
-	pushq $0; TRAP(T_RESERVED)
-IDTVEC(fpu)
-	pushq $0; TRAP(T_ARITHTRAP)
+	TRAP_ERR(T_PROTFLT)
 IDTVEC(align)
-	TRAP(T_ALIGNFLT)
-IDTVEC(xmm)
-	pushq $0; TRAP(T_XMMFLT)
+	TRAP_ERR(T_ALIGNFLT)
 	
 	/*
 	 * alltraps entry point.  Use swapgs if this is the first time in the
@@ -129,7 +147,6 @@ IDTVEC(xmm)
 	.globl	alltraps
 	.type	alltraps,@function
 alltraps:
-	subq	$TF_TRAPNO,%rsp		/* tf_err and tf_trapno already pushed */
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 	jz	alltraps_testi		/* already running with kernel GS.base */
 	swapgs
@@ -139,6 +156,7 @@ alltraps_testi:
 	sti
 alltraps_pushregs:
 	movq	%rdi,TF_RDI(%rsp)
+alltraps_pushregs_no_rdi:
 	movq	%rsi,TF_RSI(%rsp)
 	movq	%rdx,TF_RDX(%rsp)
 	movq	%rcx,TF_RCX(%rsp)
@@ -170,15 +188,14 @@ calltrap:
 	.globl	alltraps_noen
 	.type	alltraps_noen,@function
 alltraps_noen:
-	subq	$TF_TRAPNO,%rsp		/* tf_err and tf_trapno already pushed */
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 	jz	alltraps_pushregs	/* already running with kernel GS.base */
 	swapgs
 	jmp	alltraps_pushregs
 
 IDTVEC(dblfault)
-	pushq	$T_DOUBLEFLT
-	subq	$TF_TRAPNO,%rsp		/* tf_err and tf_trapno already pushed */
+	subq	$TF_ERR,%rsp
+	movq	$T_DOUBLEFLT,TF_TRAPNO(%rsp)
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 	jz	1f			/* already running with kernel GS.base */
 	swapgs
@@ -186,6 +203,20 @@ IDTVEC(dblfault)
 2:	hlt
 	jmp	2b
 
+IDTVEC(page)
+	subq	$TF_ERR,%rsp
+	movq	$T_PAGEFLT,TF_TRAPNO(%rsp)
+	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
+	jz	1f			/* already running with kernel GS.base */
+	swapgs
+1:	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */
+	movq	%cr2,%rdi		/* preserve %cr2 before ..  */
+	movq	%rdi,TF_ADDR(%rsp)	/* enabling interrupts. */
+	testl	$PSL_I,TF_RFLAGS(%rsp)
+	jz	alltraps_pushregs_no_rdi
+	sti
+	jmp	alltraps_pushregs_no_rdi
+
 /*
  * Call gate entry for FreeBSD ELF and Linux/NetBSD syscall (int 0x80)
  *
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index f828e4d..cf43159 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -152,6 +152,7 @@ ASSYM(TF_RDX, offsetof(struct trapframe, tf_rdx));
 ASSYM(TF_RCX, offsetof(struct trapframe, tf_rcx));
 ASSYM(TF_RAX, offsetof(struct trapframe, tf_rax));
 ASSYM(TF_TRAPNO, offsetof(struct trapframe, tf_trapno));
+ASSYM(TF_ADDR, offsetof(struct trapframe, tf_addr));
 ASSYM(TF_ERR, offsetof(struct trapframe, tf_err));
 ASSYM(TF_RIP, offsetof(struct trapframe, tf_rip));
 ASSYM(TF_CS, offsetof(struct trapframe, tf_cs));
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 3537170..8e7a88b 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -280,11 +280,11 @@ sendsig(catcher, sig, mask, code)
 		/* Fill in POSIX parts */
 		sf.sf_si.si_signo = sig;
 		sf.sf_si.si_code = code;
-		regs->tf_rcx = regs->tf_err;	/* arg 4 in %rcx */
+		regs->tf_rcx = regs->tf_addr;	/* arg 4 in %rcx */
 	} else {
 		/* Old FreeBSD-style arguments. */
 		regs->tf_rsi = code;		/* arg 2 in %rsi */
-		regs->tf_rcx = regs->tf_err;	/* arg 4 in %rcx */
+		regs->tf_rcx = regs->tf_addr;	/* arg 4 in %rcx */
 		sf.sf_ahu.sf_handler = catcher;
 	}
 	PROC_UNLOCK(p);
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index 19dd05b..8baa456 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -168,7 +168,7 @@ trap(frame)
 
 #ifdef DDB
 	if (db_active) {
-		eva = (type == T_PAGEFLT ? rcr2() : 0);
+		eva = (type == T_PAGEFLT ? frame.tf_addr : 0);
 		trap_fatal(&frame, eva);
 		goto out;
 	}
@@ -194,11 +194,10 @@ trap(frame)
 			printf("kernel trap %d with interrupts disabled\n",
 			    type);
 			/*
-			 * Page faults need interrupts diasabled until later,
-			 * and we shouldn't enable interrupts while holding a
+			 * We shouldn't enable interrupts while holding a
 			 * spin lock.
 			 */
-			if (type != T_PAGEFLT && PCPU_GET(spinlocks) == NULL)
+			if (PCPU_GET(spinlocks) == NULL)
 				enable_intr();
 		}
 	}
@@ -213,17 +212,9 @@ trap(frame)
 		 * do the VM lookup, so just consider it a fatal trap so the
 		 * kernel can print out a useful trap message and even get
 		 * to the debugger.
-		 *
-		 * Note that T_PAGEFLT is registered as an interrupt gate.  This
-		 * is just like a trap gate, except interrupts are disabled.  This
-		 * happens to be critically important, because we could otherwise
-		 * preempt and run another process that may cause %cr2 to be
-		 * clobbered for something else.
 		 */
-		eva = rcr2();
-		if (PCPU_GET(spinlocks) == NULL)
-			enable_intr();
-		else
+		eva = frame.tf_addr;
+		if (PCPU_GET(spinlocks) != NULL)
 			trap_fatal(&frame, eva);
 	}
 
@@ -454,7 +445,7 @@ trap(frame)
 		uprintf("fatal process exception: %s",
 			trap_msg[type]);
 		if ((type == T_PAGEFLT) || (type == T_PROTFLT))
-			uprintf(", fault VA = 0x%lx", (u_long)eva);
+			uprintf(", fault VA = 0x%lx", eva);
 		uprintf("\n");
 	}
 #endif
@@ -551,9 +542,6 @@ nogo:
 		return (-1);
 	}
 
-	/* kludge to pass faulting virtual address to sendsig */
-	frame->tf_err = eva;
-
 	return((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
 }
 
diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h
index 8f96528..09321e7 100644
--- a/sys/amd64/include/frame.h
+++ b/sys/amd64/include/frame.h
@@ -68,6 +68,7 @@ struct trapframe {
 	register_t	tf_r14;
 	register_t	tf_r15;
 	register_t	tf_trapno;
+	register_t	tf_addr;
 	/* below portion defined in hardware */
 	register_t	tf_err;
 	register_t	tf_rip;
@@ -96,6 +97,7 @@ struct intrframe {
 	register_t	if_r14;
 	register_t	if_r15;
 	register_t	:64;		/* compat with trap frame - trapno */
+	register_t	:64;		/* compat with trap frame - addr */
 	register_t	:64;		/* compat with trap frame - err */
 	/* below portion defined in hardware */
 	register_t	if_rip;
@@ -124,6 +126,7 @@ struct clockframe {
 	register_t	cf_r14;
 	register_t	cf_r15;
 	register_t	:64;		/* compat with trap frame - trapno */
+	register_t	:64;		/* compat with trap frame - addr */
 	register_t	:64;		/* compat with trap frame - err */
 	/* below portion defined in hardware */
 	register_t	cf_rip;
diff --git a/sys/amd64/include/ucontext.h b/sys/amd64/include/ucontext.h
index 7aa9d6d..ca88c0a 100644
--- a/sys/amd64/include/ucontext.h
+++ b/sys/amd64/include/ucontext.h
@@ -54,6 +54,7 @@ typedef struct __mcontext {
 	register_t	mc_r14;
 	register_t	mc_r15;
 	register_t	mc_trapno;
+	register_t	mc_addr;
 	register_t	mc_err;
 	register_t	mc_rip;
 	register_t	mc_cs;