MFamd64 r238623:

Introduce curpcb magic variable.

Requested and reviewed by:	bde
MFC after:	3 weeks

7 files changed, 55 insertions, 32 deletions

diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index 40c8c46..c194567 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -1606,7 +1606,7 @@ exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
                 pcb->pcb_dr3 = 0;
                 pcb->pcb_dr6 = 0;
                 pcb->pcb_dr7 = 0;
-                if (pcb == PCPU_GET(curpcb)) {
+                if (pcb == curpcb) {
 		        /*
 			 * Clear the debug registers on the running
 			 * CPU, otherwise they will end up affecting
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 2929789..55f6c4a 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -1962,7 +1962,7 @@ pmap_lazyfix_action(void)
 	(*ipi_lazypmap_counts[PCPU_GET(cpuid)])++;
 #endif
 	if (rcr3() == lazyptd)
-		load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+		load_cr3(curpcb->pcb_cr3);
 	CPU_CLR_ATOMIC(PCPU_GET(cpuid), lazymask);
 	atomic_store_rel_int(&lazywait, 1);
 }
@@ -1972,7 +1972,7 @@ pmap_lazyfix_self(u_int cpuid)
 {
 
 	if (rcr3() == lazyptd)
-		load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+		load_cr3(curpcb->pcb_cr3);
 	CPU_CLR_ATOMIC(cpuid, lazymask);
 }
 
@@ -2039,7 +2039,7 @@ pmap_lazyfix(pmap_t pmap)
 
 	cr3 = vtophys(pmap->pm_pdir);
 	if (cr3 == rcr3()) {
-		load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+		load_cr3(curpcb->pcb_cr3);
 		CPU_CLR(PCPU_GET(cpuid), &pmap->pm_active);
 	}
 }
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index 4a64652..c0c5678 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -344,7 +344,7 @@ trap(struct trapframe *frame)
 
         if ((ISPL(frame->tf_cs) == SEL_UPL) ||
 	    ((frame->tf_eflags & PSL_VM) && 
-		!(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL))) {
+		!(curpcb->pcb_flags & PCB_VM86CALL))) {
 		/* user trap */
 
 		td->td_pticks = 0;
@@ -593,7 +593,7 @@ trap(struct trapframe *frame)
 			/* FALL THROUGH */
 
 		case T_SEGNPFLT:	/* segment not present fault */
-			if (PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)
+			if (curpcb->pcb_flags & PCB_VM86CALL)
 				break;
 
 			/*
@@ -606,7 +606,7 @@ trap(struct trapframe *frame)
 			 * a signal.
 			 */
 			if (frame->tf_eip == (int)cpu_switch_load_gs) {
-				PCPU_GET(curpcb)->pcb_gs = 0;
+				curpcb->pcb_gs = 0;
 #if 0				
 				PROC_LOCK(p);
 				kern_psignal(p, SIGBUS);
@@ -644,9 +644,9 @@ trap(struct trapframe *frame)
 				frame->tf_eip = (int)doreti_popl_fs_fault;
 				goto out;
 			}
-			if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
+			if (curpcb->pcb_onfault != NULL) {
 				frame->tf_eip =
-				    (int)PCPU_GET(curpcb)->pcb_onfault;
+				    (int)curpcb->pcb_onfault;
 				goto out;
 			}
 			break;
@@ -696,7 +696,7 @@ trap(struct trapframe *frame)
 			 * debugging the kernel.
 			 */
 			if (user_dbreg_trap() && 
-			   !(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)) {
+			   !(curpcb->pcb_flags & PCB_VM86CALL)) {
 				/*
 				 * Reset breakpoint bits because the
 				 * processor doesn't
@@ -877,7 +877,7 @@ trap_pfault(frame, usermode, eva)
 		 * it normally, and panic immediately.
 		 */
 		if (!usermode && (td->td_intr_nesting_level != 0 ||
-		    PCPU_GET(curpcb)->pcb_onfault == NULL)) {
+		    curpcb->pcb_onfault == NULL)) {
 			trap_fatal(frame, eva);
 			return (-1);
 		}
@@ -935,8 +935,8 @@ trap_pfault(frame, usermode, eva)
 nogo:
 	if (!usermode) {
 		if (td->td_intr_nesting_level == 0 &&
-		    PCPU_GET(curpcb)->pcb_onfault != NULL) {
-			frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault;
+		    curpcb->pcb_onfault != NULL) {
+			frame->tf_eip = (int)curpcb->pcb_onfault;
 			return (0);
 		}
 		trap_fatal(frame, eva);
diff --git a/sys/i386/i386/vm86.c b/sys/i386/i386/vm86.c
index 1aaf31a..93ff855 100644
--- a/sys/i386/i386/vm86.c
+++ b/sys/i386/i386/vm86.c
@@ -143,9 +143,9 @@ vm86_emulate(vmf)
 	 * the extension is not present.  (This check should not be needed,
 	 * as we can't enter vm86 mode until we set up an extension area)
 	 */
-	if (PCPU_GET(curpcb)->pcb_ext == 0)
+	if (curpcb->pcb_ext == 0)
 		return (SIGBUS);
-	vm86 = &PCPU_GET(curpcb)->pcb_ext->ext_vm86;
+	vm86 = &curpcb->pcb_ext->ext_vm86;
 
 	if (vmf->vmf_eflags & PSL_T)
 		retcode = SIGTRAP;
@@ -535,7 +535,7 @@ vm86_prepcall(struct vm86frame *vmf)
 	vmf->kernel_fs = vmf->kernel_es = vmf->kernel_ds = 0;
 	vmf->vmf_eflags = PSL_VIF | PSL_VM | PSL_USER;
 
-	vm86 = &PCPU_GET(curpcb)->pcb_ext->ext_vm86;
+	vm86 = &curpcb->pcb_ext->ext_vm86;
 	if (!vm86->vm86_has_vme) 
 		vm86->vm86_eflags = vmf->vmf_eflags;  /* save VIF, VIP */
 }
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index 0a7bc21..dfaaa8b 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -106,6 +106,10 @@ __FBSDID("$FreeBSD$");
 #define	NSFBUFS		(512 + maxusers * 16)
 #endif
 
+CTASSERT((struct thread **)OFFSETOF_CURTHREAD ==
+    &((struct pcpu *)NULL)->pc_curthread);
+CTASSERT((struct pcb **)OFFSETOF_CURPCB == &((struct pcpu *)NULL)->pc_curpcb);
+
 static void	cpu_reset_real(void);
 #ifdef SMP
 static void	cpu_reset_proxy(void);
diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h
index aaba413..aa1066d 100644
--- a/sys/i386/include/pcpu.h
+++ b/sys/i386/include/pcpu.h
@@ -236,16 +236,36 @@ extern struct pcpu *pcpup;
 #define	PCPU_PTR(member)	__PCPU_PTR(pc_ ## member)
 #define	PCPU_SET(member, val)	__PCPU_SET(pc_ ## member, val)
 
+#define	OFFSETOF_CURTHREAD	0
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wnull-dereference"
+#endif
 static __inline __pure2 struct thread *
 __curthread(void)
 {
 	struct thread *td;
 
-	__asm("movl %%fs:0,%0" : "=r" (td));
+	__asm("movl %%fs:%1,%0" : "=r" (td)
+	    : "m" (*(char *)OFFSETOF_CURTHREAD));
 	return (td);
 }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
 #define	curthread		(__curthread())
 
+#define	OFFSETOF_CURPCB		16
+static __inline __pure2 struct pcb *
+__curpcb(void)
+{
+	struct pcb *pcb;
+
+	__asm("movl %%fs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB));
+	return (pcb);
+}
+#define	curpcb		(__curpcb())
+
 #else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */
 
 #error "this file needs to be ported to your compiler"
diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c
index 69413ff..bab0885 100644
--- a/sys/i386/isa/npx.c
+++ b/sys/i386/isa/npx.c
@@ -378,7 +378,7 @@ npxexit(td)
 
 	critical_enter();
 	if (curthread == PCPU_GET(fpcurthread))
-		npxsave(PCPU_GET(curpcb)->pcb_save);
+		npxsave(curpcb->pcb_save);
 	critical_exit();
 #ifdef NPX_DEBUG
 	if (hw_float) {
@@ -663,7 +663,6 @@ static int err_count = 0;
 int
 npxdna(void)
 {
-	struct pcb *pcb;
 
 	if (!hw_float)
 		return (0);
@@ -687,25 +686,24 @@ npxdna(void)
 	 * Record new context early in case frstor causes an IRQ13.
 	 */
 	PCPU_SET(fpcurthread, curthread);
-	pcb = PCPU_GET(curpcb);
 
 #ifdef CPU_ENABLE_SSE
 	if (cpu_fxsr)
 		fpu_clean_state();
 #endif
 
-	if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
+	if ((curpcb->pcb_flags & PCB_NPXINITDONE) == 0) {
 		/*
 		 * This is the first time this thread has used the FPU or
 		 * the PCB doesn't contain a clean FPU state.  Explicitly
 		 * load an initial state.
 		 */
 		fpurstor(&npx_initialstate);
-		if (pcb->pcb_initial_npxcw != __INITIAL_NPXCW__)
-			fldcw(pcb->pcb_initial_npxcw);
-		pcb->pcb_flags |= PCB_NPXINITDONE;
-		if (PCB_USER_FPU(pcb))
-			pcb->pcb_flags |= PCB_NPXUSERINITDONE;
+		if (curpcb->pcb_initial_npxcw != __INITIAL_NPXCW__)
+			fldcw(curpcb->pcb_initial_npxcw);
+		curpcb->pcb_flags |= PCB_NPXINITDONE;
+		if (PCB_USER_FPU(curpcb))
+			curpcb->pcb_flags |= PCB_NPXUSERINITDONE;
 	} else {
 		/*
 		 * The following fpurstor() may cause an IRQ13 when the
@@ -721,7 +719,7 @@ npxdna(void)
 		 * fnclex if it is the first FPU instruction after a context
 		 * switch.
 		 */
-		fpurstor(pcb->pcb_save);
+		fpurstor(curpcb->pcb_save);
 	}
 	critical_exit();
 
@@ -1099,13 +1097,14 @@ fpu_kern_thread(u_int flags)
 {
 	struct pcb *pcb;
 
-	pcb = PCPU_GET(curpcb);
+	pcb = curpcb;
 	KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
 	    ("Only kthread may use fpu_kern_thread"));
-	KASSERT(pcb->pcb_save == &pcb->pcb_user_save, ("mangled pcb_save"));
-	KASSERT(PCB_USER_FPU(pcb), ("recursive call"));
+	KASSERT(curpcb->pcb_save == &curpcb->pcb_user_save,
+	    ("mangled pcb_save"));
+	KASSERT(PCB_USER_FPU(curpcb), ("recursive call"));
 
-	pcb->pcb_flags |= PCB_KERNNPX;
+	curpcb->pcb_flags |= PCB_KERNNPX;
 	return (0);
 }
 
@@ -1115,5 +1114,5 @@ is_fpu_kern_thread(u_int flags)
 
 	if ((curthread->td_pflags & TDP_KTHREAD) == 0)
 		return (0);
-	return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNNPX) != 0);
+	return ((curpcb->pcb_flags & PCB_KERNNPX) != 0);
 }