Initial landing of SMP support for FreeBSD/amd64.

- This is heavily derived from John Baldwin's apic/pci cleanup on i386.
- I have completely rewritten or drastically cleaned up some other parts.
  (in particular, bootstrap)
- This is still a WIP.  It seems that there are some highly bogus bioses
  on nVidia nForce3-150 boards.  I can't stress how broken these boards
  are.  I have a workaround in mind, but right now the Asus SK8N is broken.
  The Gigabyte K8NPro (nVidia based) is also mind-numbingly hosed.
- Most of my testing has been with SCHED_ULE.  SCHED_4BSD works.
- the apic and acpi components are 'standard'.
- If you have an nVidia nForce3-150 board, you are stuck with 'device
  atpic' in addition, because they somehow managed to forget to connect the
  8254 timer to the apic, even though its in the same silicon!  ARGH!
  This directly violates the ACPI spec.

1 files changed, 103 insertions, 173 deletions

diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index ecc4c67..2160dc4 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -42,7 +42,6 @@
 
 #include <machine/asmacros.h>
 #include <machine/apicreg.h>
-#include <machine/smptests.h>
 
 #include "assym.s"
 
@@ -50,19 +49,48 @@
  * Macros to create and destroy a trap frame.
  */
 #define PUSH_FRAME							\
-	pushl	$0 ;		/* dummy error code */			\
-	pushl	$0 ;		/* dummy trap type */			\
-	pushal ;		/* 8 ints */				\
-	pushl	%ds ;		/* save data and extra segments ... */	\
-	pushl	%es ;							\
-	pushl	%fs
+	subq	$TF_RIP,%rsp ;	/* skip dummy tf_err and tf_trapno */	\
+	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
+	jz	1f ;		/* Yes, dont swapgs again */		\
+	swapgs ;							\
+1:	movq	%rdi,TF_RDI(%rsp) ;					\
+	movq	%rsi,TF_RSI(%rsp) ;					\
+	movq	%rdx,TF_RDX(%rsp) ;					\
+	movq	%rcx,TF_RCX(%rsp) ;					\
+	movq	%r8,TF_R8(%rsp) ;					\
+	movq	%r9,TF_R9(%rsp) ;					\
+	movq	%rax,TF_RAX(%rsp) ;					\
+	movq	%rbx,TF_RBX(%rsp) ;					\
+	movq	%rbp,TF_RBP(%rsp) ;					\
+	movq	%r10,TF_R10(%rsp) ;					\
+	movq	%r11,TF_R11(%rsp) ;					\
+	movq	%r12,TF_R12(%rsp) ;					\
+	movq	%r13,TF_R13(%rsp) ;					\
+	movq	%r14,TF_R14(%rsp) ;					\
+	movq	%r15,TF_R15(%rsp)
 
 #define POP_FRAME							\
-	popl	%fs ;							\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp
+	movq	TF_RDI(%rsp),%rdi ;					\
+	movq	TF_RSI(%rsp),%rsi ;					\
+	movq	TF_RDX(%rsp),%rdx ;					\
+	movq	TF_RCX(%rsp),%rcx ;					\
+	movq	TF_R8(%rsp),%r8 ;					\
+	movq	TF_R9(%rsp),%r9 ;					\
+	movq	TF_RAX(%rsp),%rax ;					\
+	movq	TF_RBX(%rsp),%rbx ;					\
+	movq	TF_RBP(%rsp),%rbp ;					\
+	movq	TF_R10(%rsp),%r10 ;					\
+	movq	TF_R11(%rsp),%r11 ;					\
+	movq	TF_R12(%rsp),%r12 ;					\
+	movq	TF_R13(%rsp),%r13 ;					\
+	movq	TF_R14(%rsp),%r14 ;					\
+	movq	TF_R15(%rsp),%r15 ;					\
+	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
+	jz	1f ;		/* keep kernel GS.base */		\
+	cli ;								\
+	swapgs ;							\
+1:	addq	$TF_RIP,%rsp	/* skip over tf_err, tf_trapno */
+
 
 /*
  * I/O Interrupt Entry Point.  Rather than having one entry point for
@@ -76,21 +104,15 @@
 	SUPERALIGN_TEXT ;						\
 IDTVEC(vec_name) ;							\
 	PUSH_FRAME ;							\
-	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
-	mov	%ax, %ds ;						\
-	mov	%ax, %es ;						\
-	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
-	mov	%ax, %fs ;						\
-	movl	lapic, %edx ;	/* pointer to local APIC */		\
-	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
+	movq	lapic, %rdx ;	/* pointer to local APIC */		\
+	movl	LA_ISR + 16 * (index)(%rdx), %eax ;	/* load ISR */	\
 	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
 	jz	2f ;							\
 	addl	$(32 * index),%eax ;					\
 1: ;									\
 	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX avoid double count */	\
-	pushl	%eax ;		/* pass the IRQ */			\
+	movq	%rax, %rdi ;	/* pass the IRQ */			\
 	call	lapic_handle_intr ;					\
-	addl	$4, %esp ;	/* discard parameter */			\
 	MEXITCOUNT ;							\
 	jmp	doreti ;						\
 2:	movl	$-1, %eax ;	/* send a vector of -1 */		\
@@ -109,7 +131,7 @@ IDTVEC(spuriousint)
 
 	/* No EOI cycle used here */
 
-	iret
+	iretq
 
 MCOUNT_LABEL(bintr2)
 	ISR_VEC(1, apic_isr1)
@@ -128,32 +150,19 @@ MCOUNT_LABEL(eintr2)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invltlb)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	mov	%ax, %ds
-
-#ifdef COUNT_XINVLTLB_HITS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	mov	%ax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	incl	xhits_gbl(,%eax,4)
-#endif /* COUNT_XINVLTLB_HITS */
+	pushq	%rax
 
-	movl	%cr3, %eax		/* invalidate the TLB */
-	movl	%eax, %cr3
+	movq	%cr3, %rax		/* invalidate the TLB */
+	movq	%rax, %cr3
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
 
-	popl	%ds
-	popl	%eax
-	iret
+	popq	%rax
+	iretq
 
 /*
  * Single page TLB shootdown
@@ -161,32 +170,19 @@ IDTVEC(invltlb)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlpg)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	mov	%ax, %ds
-
-#ifdef COUNT_XINVLTLB_HITS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	mov	%ax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	incl	xhits_pg(,%eax,4)
-#endif /* COUNT_XINVLTLB_HITS */
+	pushq	%rax
 
-	movl	smp_tlb_addr1, %eax
-	invlpg	(%eax)			/* invalidate single page */
+	movq	smp_tlb_addr1, %rax
+	invlpg	(%rax)			/* invalidate single page */
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
 
-	popl	%ds
-	popl	%eax
-	iret
+	popq	%rax
+	iretq
 
 /*
  * Page range TLB shootdown.
@@ -194,38 +190,25 @@ IDTVEC(invlpg)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlrng)
-	pushl	%eax
-	pushl	%edx
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	mov	%ax, %ds
-
-#ifdef COUNT_XINVLTLB_HITS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	mov	%ax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	incl	xhits_rng(,%eax,4)
-#endif /* COUNT_XINVLTLB_HITS */
-
-	movl	smp_tlb_addr1, %edx
-	movl	smp_tlb_addr2, %eax
-1:	invlpg	(%edx)			/* invalidate single page */
-	addl	$PAGE_SIZE, %edx
-	cmpl	%eax, %edx
+	pushq	%rax
+	pushq	%rdx
+
+	movq	smp_tlb_addr1, %rdx
+	movq	smp_tlb_addr2, %rax
+1:	invlpg	(%rdx)			/* invalidate single page */
+	addq	$PAGE_SIZE, %rdx
+	cmpq	%rax, %rdx
 	jb	1b
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
 
-	popl	%ds
-	popl	%edx
-	popl	%eax
-	iret
+	popq	%rdx
+	popq	%rax
+	iretq
 
 /*
  * Forward hardclock to another CPU.  Pushes a clockframe and calls
@@ -235,18 +218,11 @@ IDTVEC(invlrng)
 	SUPERALIGN_TEXT
 IDTVEC(hardclock)
 	PUSH_FRAME
-	movl	$KDSEL, %eax	/* reload with kernel's data segment */
-	mov	%ax, %ds
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
 
-	movl	lapic, %edx
-	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
+	movq	lapic, %rdx
+	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
 
-	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_hardclock
-	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	MEXITCOUNT
 	jmp	doreti
 
@@ -258,20 +234,13 @@ IDTVEC(hardclock)
 	SUPERALIGN_TEXT
 IDTVEC(statclock)
 	PUSH_FRAME
-	movl	$KDSEL, %eax	/* reload with kernel's data segment */
-	mov	%ax, %ds
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
 
-	movl	lapic, %edx
-	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
+	movq	lapic, %rdx
+	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
-	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_statclock
-	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	MEXITCOUNT
 	jmp	doreti
 
@@ -287,14 +256,9 @@ IDTVEC(statclock)
 	SUPERALIGN_TEXT
 IDTVEC(cpuast)
 	PUSH_FRAME
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
 
-	movl	lapic, %edx
-	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
+	movq	lapic, %rdx
+	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
@@ -311,63 +275,41 @@ IDTVEC(cpuast)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(cpustop)
-	pushl	%ebp
-	movl	%esp, %ebp
-	pushl	%eax
-	pushl	%ecx
-	pushl	%edx
-	pushl	%ds			/* save current data segment */
-	pushl	%es
-	pushl	%fs
-
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	PUSH_FRAME
+
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	movl	PCPU(CPUID), %eax
 	imull	$PCB_SIZE, %eax
-	leal	CNAME(stoppcbs)(%eax), %eax
-	pushl	%eax
-	call	CNAME(savectx)		/* Save process context */
-	addl	$4, %esp
+	leaq	stoppcbs(%rax), %rdi
+	call	savectx			/* Save process context */
 		
 	movl	PCPU(CPUID), %eax
 
 	lock
-	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
+	btsl	%eax, stopped_cpus	/* stopped_cpus |= (1<<id) */
 1:
-	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
+	btl	%eax, started_cpus	/* while (!(started_cpus & (1<<id))) */
 	jnc	1b
 
 	lock
-	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
+	btrl	%eax, started_cpus	/* started_cpus &= ~(1<<id) */
 	lock
-	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
+	btrl	%eax, stopped_cpus	/* stopped_cpus &= ~(1<<id) */
 
 	test	%eax, %eax
 	jnz	2f
 
-	movl	CNAME(cpustop_restartfunc), %eax
-	test	%eax, %eax
+	movq	cpustop_restartfunc, %rax
+	testq	%rax, %rax
 	jz	2f
-	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
+	movq	$0, cpustop_restartfunc	/* One-shot */
 
-	call	*%eax
+	call	*%rax
 2:
-	popl	%fs
-	popl	%es
-	popl	%ds			/* restore previous data segment */
-	popl	%edx
-	popl	%ecx
-	popl	%eax
-	movl	%ebp, %esp
-	popl	%ebp
-	iret
+	POP_FRAME
+	iretq
 
 /*
  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
@@ -378,19 +320,13 @@ IDTVEC(cpustop)
 	SUPERALIGN_TEXT
 IDTVEC(rendezvous)
 	PUSH_FRAME
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
-
 	call	smp_rendezvous_action
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
-	POP_FRAME
-	iret
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	POP_FRAME			/* Why not doreti? */
+	iretq
 	
+#ifdef LAZY_SWITCH
 /*
  * Clean up when we lose out on the lazy context switch optimization.
  * ie: when we are about to release a PTD but a cpu is still borrowing it.
@@ -398,16 +334,10 @@ IDTVEC(rendezvous)
 	SUPERALIGN_TEXT
 IDTVEC(lazypmap)
 	PUSH_FRAME
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
-
 	call	pmap_lazyfix_action
-
-	movl	lapic, %eax	
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
-	POP_FRAME
-	iret
+	movq	lapic, %rax	
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	POP_FRAME			/* Why not doreti? */
+	iretq
+#endif
 #endif /* SMP */