MFC 270850,271053,271192,271717:

Save and restore FPU state across suspend and resume on i386.
- Create a separate structure for per-CPU state saved across suspend and
  resume that is a superset of a pcb.
- Store the FPU state for suspend and resume in the new structure
  (for amd64, this moves it out of the PCB)
- On both i386 and amd64, all of the FPU suspend/resume handling is now
  done in C.

Approved by:	re (hrs)

6 files changed, 29 insertions, 38 deletions

diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 7c37a41..eb0ee8b 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -399,10 +399,6 @@ ENTRY(savectx)
 	rdmsr
 	movl	%eax,PCB_SFMASK(%rdi)
 	movl	%edx,PCB_SFMASK+4(%rdi)
-	movl	xsave_mask,%eax
-	movl	%eax,PCB_XSMASK(%rdi)
-	movl	xsave_mask+4,%eax
-	movl	%eax,PCB_XSMASK+4(%rdi)
 
 	sgdt	PCB_GDT(%rdi)
 	sidt	PCB_IDT(%rdi)
@@ -467,12 +463,9 @@ ENTRY(resumectx)
 	movl	PCB_SFMASK(%rdi),%eax
 	wrmsr
 
-	/* Restore CR0 except for FPU mode. */
+	/* Restore CR0, CR2, CR4 and CR3. */
 	movq	PCB_CR0(%rdi),%rax
-	andq	$~(CR0_EM | CR0_TS),%rax
 	movq	%rax,%cr0
-
-	/* Restore CR2, CR4 and CR3. */
 	movq	PCB_CR2(%rdi),%rax
 	movq	%rax,%cr2
 	movq	PCB_CR4(%rdi),%rax
@@ -510,26 +503,6 @@ ENTRY(resumectx)
 	movq	PCB_DR7(%rdi),%rax
 	movq	%rax,%dr7
 
-	/* Restore FPU state. */
-	fninit
-	movq	PCB_FPUSUSPEND(%rdi),%rbx
-	movq	PCB_XSMASK(%rdi),%rax
-	testq	%rax,%rax
-	jz	1f
-	movq	%rax,%rdx
-	shrq	$32,%rdx
-	movl	$XCR0,%ecx
-	xsetbv
-	xrstor	(%rbx)
-	jmp	2f
-1:
-	fxrstor	(%rbx)
-2:
-
-	/* Reload CR0. */
-	movq	PCB_CR0(%rdi),%rax
-	movq	%rax,%cr0
-
 	/* Restore other callee saved registers. */
 	movq	PCB_R15(%rdi),%r15
 	movq	PCB_R14(%rdi),%r14
diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c
index 0f4b2b5..fef47a0 100644
--- a/sys/amd64/amd64/fpu.c
+++ b/sys/amd64/amd64/fpu.c
@@ -173,6 +173,20 @@ fpususpend(void *addr)
 	load_cr0(cr0);
 }
 
+void
+fpuresume(void *addr)
+{
+	u_long cr0;
+
+	cr0 = rcr0();
+	stop_emulating();
+	fninit();
+	if (use_xsave)
+		load_xcr(XCR0, xsave_mask);
+	fpurestore(addr);
+	load_cr0(cr0);
+}
+
 /*
  * Enable XSAVE if supported and allowed by user.
  * Calculate the xsave_mask.
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 028a2cd..09696ef 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -163,8 +163,6 @@ ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star));
 ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar));
 ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar));
 ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask));
-ASSYM(PCB_XSMASK, offsetof(struct pcb, pcb_xsmask));
-ASSYM(PCB_FPUSUSPEND, offsetof(struct pcb, pcb_fpususpend));
 ASSYM(PCB_SIZE, sizeof(struct pcb));
 ASSYM(PCB_FULL_IRET, PCB_FULL_IRET);
 ASSYM(PCB_DBREGS, PCB_DBREGS);
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index e09730d..629ffaa 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -101,7 +101,7 @@ char *nmi_stack;
 void *dpcpu;
 
 struct pcb stoppcbs[MAXCPU];
-struct pcb **susppcbs;
+struct susppcb **susppcbs;
 
 /* Variables needed for SMP tlb shootdown. */
 vm_offset_t smp_tlb_addr2;
@@ -1463,11 +1463,12 @@ cpususpend_handler(void)
 	mtx_assert(&smp_ipi_mtx, MA_NOTOWNED);
 
 	cpu = PCPU_GET(cpuid);
-	if (savectx(susppcbs[cpu])) {
-		fpususpend(susppcbs[cpu]->pcb_fpususpend);
+	if (savectx(&susppcbs[cpu]->sp_pcb)) {
+		fpususpend(susppcbs[cpu]->sp_fpususpend);
 		wbinvd();
 		CPU_SET_ATOMIC(cpu, &suspended_cpus);
 	} else {
+		fpuresume(susppcbs[cpu]->sp_fpususpend);
 		pmap_init_pat();
 		initializecpu();
 		PCPU_SET(switchtime, 0);
diff --git a/sys/amd64/include/fpu.h b/sys/amd64/include/fpu.h
index be1bdc6..78d2fee 100644
--- a/sys/amd64/include/fpu.h
+++ b/sys/amd64/include/fpu.h
@@ -58,6 +58,7 @@ int	fpuformat(void);
 int	fpugetregs(struct thread *td);
 void	fpuinit(void);
 void	fpurestore(void *addr);
+void	fpuresume(void *addr);
 void	fpusave(void *addr);
 int	fpusetregs(struct thread *td, struct savefpu *addr,
 	    char *xfpustate, size_t xfpustate_size);
diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h
index 80aff86..153393f 100644
--- a/sys/amd64/include/pcb.h
+++ b/sys/amd64/include/pcb.h
@@ -97,14 +97,18 @@ struct pcb {
 	register_t	pcb_lstar;
 	register_t	pcb_cstar;
 	register_t	pcb_sfmask;
-	register_t	pcb_xsmask;
-
-	/* fpu context for suspend/resume */
-	void		*pcb_fpususpend;
 
 	struct savefpu	*pcb_save;
 
-	uint64_t	pcb_pad[3];
+	uint64_t	pcb_pad[5];
+};
+
+/* Per-CPU state saved during suspend and resume. */
+struct susppcb {
+	struct pcb	sp_pcb;
+
+	/* fpu context for suspend/resume */
+	void		*sp_fpususpend;
 };
 #endif