Initial landing of SMP support for FreeBSD/amd64.

- This is heavily derived from John Baldwin's apic/pci cleanup on i386.
- I have completely rewritten or drastically cleaned up some other parts.
  (in particular, bootstrap)
- This is still a WIP.  It seems that there are some highly bogus bioses
  on nVidia nForce3-150 boards.  I can't stress how broken these boards
  are.  I have a workaround in mind, but right now the Asus SK8N is broken.
  The Gigabyte K8NPro (nVidia based) is also mind-numbingly hosed.
- Most of my testing has been with SCHED_ULE.  SCHED_4BSD works.
- the apic and acpi components are 'standard'.
- If you have an nVidia nForce3-150 board, you are stuck with 'device
  atpic' in addition, because they somehow managed to forget to connect the
  8254 timer to the apic, even though its in the same silicon!  ARGH!
  This directly violates the ACPI spec.

50 files changed, 1113 insertions, 2298 deletions

diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index ecc4c67..2160dc4 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -42,7 +42,6 @@
 
 #include <machine/asmacros.h>
 #include <machine/apicreg.h>
-#include <machine/smptests.h>
 
 #include "assym.s"
 
@@ -50,19 +49,48 @@
  * Macros to create and destroy a trap frame.
  */
 #define PUSH_FRAME							\
-	pushl	$0 ;		/* dummy error code */			\
-	pushl	$0 ;		/* dummy trap type */			\
-	pushal ;		/* 8 ints */				\
-	pushl	%ds ;		/* save data and extra segments ... */	\
-	pushl	%es ;							\
-	pushl	%fs
+	subq	$TF_RIP,%rsp ;	/* skip dummy tf_err and tf_trapno */	\
+	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
+	jz	1f ;		/* Yes, dont swapgs again */		\
+	swapgs ;							\
+1:	movq	%rdi,TF_RDI(%rsp) ;					\
+	movq	%rsi,TF_RSI(%rsp) ;					\
+	movq	%rdx,TF_RDX(%rsp) ;					\
+	movq	%rcx,TF_RCX(%rsp) ;					\
+	movq	%r8,TF_R8(%rsp) ;					\
+	movq	%r9,TF_R9(%rsp) ;					\
+	movq	%rax,TF_RAX(%rsp) ;					\
+	movq	%rbx,TF_RBX(%rsp) ;					\
+	movq	%rbp,TF_RBP(%rsp) ;					\
+	movq	%r10,TF_R10(%rsp) ;					\
+	movq	%r11,TF_R11(%rsp) ;					\
+	movq	%r12,TF_R12(%rsp) ;					\
+	movq	%r13,TF_R13(%rsp) ;					\
+	movq	%r14,TF_R14(%rsp) ;					\
+	movq	%r15,TF_R15(%rsp)
 
 #define POP_FRAME							\
-	popl	%fs ;							\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp
+	movq	TF_RDI(%rsp),%rdi ;					\
+	movq	TF_RSI(%rsp),%rsi ;					\
+	movq	TF_RDX(%rsp),%rdx ;					\
+	movq	TF_RCX(%rsp),%rcx ;					\
+	movq	TF_R8(%rsp),%r8 ;					\
+	movq	TF_R9(%rsp),%r9 ;					\
+	movq	TF_RAX(%rsp),%rax ;					\
+	movq	TF_RBX(%rsp),%rbx ;					\
+	movq	TF_RBP(%rsp),%rbp ;					\
+	movq	TF_R10(%rsp),%r10 ;					\
+	movq	TF_R11(%rsp),%r11 ;					\
+	movq	TF_R12(%rsp),%r12 ;					\
+	movq	TF_R13(%rsp),%r13 ;					\
+	movq	TF_R14(%rsp),%r14 ;					\
+	movq	TF_R15(%rsp),%r15 ;					\
+	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
+	jz	1f ;		/* keep kernel GS.base */		\
+	cli ;								\
+	swapgs ;							\
+1:	addq	$TF_RIP,%rsp	/* skip over tf_err, tf_trapno */
+
 
 /*
  * I/O Interrupt Entry Point.  Rather than having one entry point for
@@ -76,21 +104,15 @@
 	SUPERALIGN_TEXT ;						\
 IDTVEC(vec_name) ;							\
 	PUSH_FRAME ;							\
-	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
-	mov	%ax, %ds ;						\
-	mov	%ax, %es ;						\
-	movl	$KPSEL, %eax ;	/* reload with per-CPU data segment */	\
-	mov	%ax, %fs ;						\
-	movl	lapic, %edx ;	/* pointer to local APIC */		\
-	movl	LA_ISR + 16 * (index)(%edx), %eax ;	/* load ISR */	\
+	movq	lapic, %rdx ;	/* pointer to local APIC */		\
+	movl	LA_ISR + 16 * (index)(%rdx), %eax ;	/* load ISR */	\
 	bsrl	%eax, %eax ;	/* index of highset set bit in ISR */	\
 	jz	2f ;							\
 	addl	$(32 * index),%eax ;					\
 1: ;									\
 	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX avoid double count */	\
-	pushl	%eax ;		/* pass the IRQ */			\
+	movq	%rax, %rdi ;	/* pass the IRQ */			\
 	call	lapic_handle_intr ;					\
-	addl	$4, %esp ;	/* discard parameter */			\
 	MEXITCOUNT ;							\
 	jmp	doreti ;						\
 2:	movl	$-1, %eax ;	/* send a vector of -1 */		\
@@ -109,7 +131,7 @@ IDTVEC(spuriousint)
 
 	/* No EOI cycle used here */
 
-	iret
+	iretq
 
 MCOUNT_LABEL(bintr2)
 	ISR_VEC(1, apic_isr1)
@@ -128,32 +150,19 @@ MCOUNT_LABEL(eintr2)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invltlb)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	mov	%ax, %ds
-
-#ifdef COUNT_XINVLTLB_HITS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	mov	%ax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	incl	xhits_gbl(,%eax,4)
-#endif /* COUNT_XINVLTLB_HITS */
+	pushq	%rax
 
-	movl	%cr3, %eax		/* invalidate the TLB */
-	movl	%eax, %cr3
+	movq	%cr3, %rax		/* invalidate the TLB */
+	movq	%rax, %cr3
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
 
-	popl	%ds
-	popl	%eax
-	iret
+	popq	%rax
+	iretq
 
 /*
  * Single page TLB shootdown
@@ -161,32 +170,19 @@ IDTVEC(invltlb)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlpg)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	mov	%ax, %ds
-
-#ifdef COUNT_XINVLTLB_HITS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	mov	%ax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	incl	xhits_pg(,%eax,4)
-#endif /* COUNT_XINVLTLB_HITS */
+	pushq	%rax
 
-	movl	smp_tlb_addr1, %eax
-	invlpg	(%eax)			/* invalidate single page */
+	movq	smp_tlb_addr1, %rax
+	invlpg	(%rax)			/* invalidate single page */
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
 
-	popl	%ds
-	popl	%eax
-	iret
+	popq	%rax
+	iretq
 
 /*
  * Page range TLB shootdown.
@@ -194,38 +190,25 @@ IDTVEC(invlpg)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlrng)
-	pushl	%eax
-	pushl	%edx
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	mov	%ax, %ds
-
-#ifdef COUNT_XINVLTLB_HITS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	mov	%ax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	incl	xhits_rng(,%eax,4)
-#endif /* COUNT_XINVLTLB_HITS */
-
-	movl	smp_tlb_addr1, %edx
-	movl	smp_tlb_addr2, %eax
-1:	invlpg	(%edx)			/* invalidate single page */
-	addl	$PAGE_SIZE, %edx
-	cmpl	%eax, %edx
+	pushq	%rax
+	pushq	%rdx
+
+	movq	smp_tlb_addr1, %rdx
+	movq	smp_tlb_addr2, %rax
+1:	invlpg	(%rdx)			/* invalidate single page */
+	addq	$PAGE_SIZE, %rdx
+	cmpq	%rax, %rdx
 	jb	1b
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	lock
 	incl	smp_tlb_wait
 
-	popl	%ds
-	popl	%edx
-	popl	%eax
-	iret
+	popq	%rdx
+	popq	%rax
+	iretq
 
 /*
  * Forward hardclock to another CPU.  Pushes a clockframe and calls
@@ -235,18 +218,11 @@ IDTVEC(invlrng)
 	SUPERALIGN_TEXT
 IDTVEC(hardclock)
 	PUSH_FRAME
-	movl	$KDSEL, %eax	/* reload with kernel's data segment */
-	mov	%ax, %ds
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
 
-	movl	lapic, %edx
-	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
+	movq	lapic, %rdx
+	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
 
-	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_hardclock
-	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	MEXITCOUNT
 	jmp	doreti
 
@@ -258,20 +234,13 @@ IDTVEC(hardclock)
 	SUPERALIGN_TEXT
 IDTVEC(statclock)
 	PUSH_FRAME
-	movl	$KDSEL, %eax	/* reload with kernel's data segment */
-	mov	%ax, %ds
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
 
-	movl	lapic, %edx
-	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
+	movq	lapic, %rdx
+	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
-	pushl	$0		/* XXX convert trapframe to clockframe */
 	call	forwarded_statclock
-	addl	$4, %esp	/* XXX convert clockframe to trapframe */
 	MEXITCOUNT
 	jmp	doreti
 
@@ -287,14 +256,9 @@ IDTVEC(statclock)
 	SUPERALIGN_TEXT
 IDTVEC(cpuast)
 	PUSH_FRAME
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
 
-	movl	lapic, %edx
-	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
+	movq	lapic, %rdx
+	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
 
 	FAKE_MCOUNT(13*4(%esp))
 
@@ -311,63 +275,41 @@ IDTVEC(cpuast)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(cpustop)
-	pushl	%ebp
-	movl	%esp, %ebp
-	pushl	%eax
-	pushl	%ecx
-	pushl	%edx
-	pushl	%ds			/* save current data segment */
-	pushl	%es
-	pushl	%fs
-
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	PUSH_FRAME
+
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	movl	PCPU(CPUID), %eax
 	imull	$PCB_SIZE, %eax
-	leal	CNAME(stoppcbs)(%eax), %eax
-	pushl	%eax
-	call	CNAME(savectx)		/* Save process context */
-	addl	$4, %esp
+	leaq	stoppcbs(%rax), %rdi
+	call	savectx			/* Save process context */
 		
 	movl	PCPU(CPUID), %eax
 
 	lock
-	btsl	%eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
+	btsl	%eax, stopped_cpus	/* stopped_cpus |= (1<<id) */
 1:
-	btl	%eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
+	btl	%eax, started_cpus	/* while (!(started_cpus & (1<<id))) */
 	jnc	1b
 
 	lock
-	btrl	%eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
+	btrl	%eax, started_cpus	/* started_cpus &= ~(1<<id) */
 	lock
-	btrl	%eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
+	btrl	%eax, stopped_cpus	/* stopped_cpus &= ~(1<<id) */
 
 	test	%eax, %eax
 	jnz	2f
 
-	movl	CNAME(cpustop_restartfunc), %eax
-	test	%eax, %eax
+	movq	cpustop_restartfunc, %rax
+	testq	%rax, %rax
 	jz	2f
-	movl	$0, CNAME(cpustop_restartfunc)	/* One-shot */
+	movq	$0, cpustop_restartfunc	/* One-shot */
 
-	call	*%eax
+	call	*%rax
 2:
-	popl	%fs
-	popl	%es
-	popl	%ds			/* restore previous data segment */
-	popl	%edx
-	popl	%ecx
-	popl	%eax
-	movl	%ebp, %esp
-	popl	%ebp
-	iret
+	POP_FRAME
+	iretq
 
 /*
  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
@@ -378,19 +320,13 @@ IDTVEC(cpustop)
 	SUPERALIGN_TEXT
 IDTVEC(rendezvous)
 	PUSH_FRAME
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
-
 	call	smp_rendezvous_action
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
-	POP_FRAME
-	iret
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	POP_FRAME			/* Why not doreti? */
+	iretq
 	
+#ifdef LAZY_SWITCH
 /*
  * Clean up when we lose out on the lazy context switch optimization.
  * ie: when we are about to release a PTD but a cpu is still borrowing it.
@@ -398,16 +334,10 @@ IDTVEC(rendezvous)
 	SUPERALIGN_TEXT
 IDTVEC(lazypmap)
 	PUSH_FRAME
-	movl	$KDSEL, %eax
-	mov	%ax, %ds		/* use KERNEL data segment */
-	mov	%ax, %es
-	movl	$KPSEL, %eax
-	mov	%ax, %fs
-
 	call	pmap_lazyfix_action
-
-	movl	lapic, %eax	
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
-	POP_FRAME
-	iret
+	movq	lapic, %rax	
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+	POP_FRAME			/* Why not doreti? */
+	iretq
+#endif
 #endif /* SMP */
diff --git a/sys/amd64/amd64/autoconf.c b/sys/amd64/amd64/autoconf.c
index d6ce6b6..adec2e0 100644
--- a/sys/amd64/amd64/autoconf.c
+++ b/sys/amd64/amd64/autoconf.c
@@ -76,7 +76,6 @@ __FBSDID("$FreeBSD$");
 #include <nfsclient/nfsdiskless.h>
 
 #include <machine/md_var.h>
-#include <amd64/isa/icu.h>
 
 #ifdef DEV_ISA
 #include <isa/isavar.h>
@@ -109,23 +108,11 @@ configure(dummy)
 {
 
 	/*
-	 * Activate the ICU's.  Note that we are explicitly at splhigh()
-	 * at present as we have no way to disable stray PCI level triggered
-	 * interrupts until the devices have had a driver attached.  This
-	 * is particularly a problem when the interrupts are shared.  For
-	 * example, if IRQ 10 is shared between a disk and network device
-	 * and the disk device generates an interrupt, if we "activate"
-	 * IRQ 10 when the network driver is set up, then we will get
-	 * recursive interrupt 10's as nothing will know how to turn off
-	 * the disk device's interrupt.
-	 *
-	 * Having the ICU's active means we can probe interrupt routing to
-	 * see if a device causes the corresponding pending bit to be set.
-	 *
-	 * This is all rather inconvenient.
+	 * Enable interrupts on the processor.  The interrupts are still
+	 * disabled in the interrupt controllers until interrupt handlers
+	 * are registered.
 	 */
 	enable_intr();
-	INTREN(IRQ_SLAVE);
 
 	/* nexus0 is the top of the i386 device tree */
 	device_add_child(root_bus, "nexus", 0);
@@ -141,12 +128,6 @@ configure(dummy)
 	if (isa_bus_device)
 		isa_probe_children(isa_bus_device);
 #endif
-
-	/*
-	 * Now we're ready to handle (pending) interrupts.
-	 * XXX this is slightly misplaced.
-	 */
-	spl0();
 }
 
 static void
diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 56f0c84..3bfcfc8 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -59,14 +59,16 @@
  * %rsi = newtd
  */
 ENTRY(cpu_throw)
-	xorq	%rax, %rax
 	movl	PCPU(CPUID), %eax
 	testq	%rdi,%rdi			/* no thread? */
 	jz	1f
 	/* release bit from old pm_active */
 	movq	TD_PROC(%rdi), %rdx		/* oldtd->td_proc */
 	movq	P_VMSPACE(%rdx), %rdx		/* proc->p_vmspace */
-	btrq	%rax, VM_PMAP+PM_ACTIVE(%rdx)	/* clear old */
+#ifdef SMP
+	lock
+#endif
+	btrl	%eax, VM_PMAP+PM_ACTIVE(%rdx)	/* clear old */
 1:
 	movq	TD_PCB(%rsi),%rdx		/* newtd->td_proc */
 	movq	PCB_CR3(%rdx),%rdx
@@ -74,7 +76,10 @@ ENTRY(cpu_throw)
 	/* set bit in new pm_active */
 	movq	TD_PROC(%rsi),%rdx
 	movq	P_VMSPACE(%rdx), %rdx
-	btsq	%rax, VM_PMAP+PM_ACTIVE(%rdx)	/* set new */
+#ifdef SMP
+	lock
+#endif
+	btsl	%eax, VM_PMAP+PM_ACTIVE(%rdx)	/* set new */
 	jmp	sw1
 
 /*
@@ -143,7 +148,6 @@ ENTRY(cpu_switch)
 	jz	badsw3				/* no, panic */
 #endif
 	movq	TD_PCB(%rsi),%r8
-	xorq	%rax, %rax
 	movl	PCPU(CPUID), %eax
 
 	/* switch address space */
@@ -153,12 +157,18 @@ ENTRY(cpu_switch)
 	/* Release bit from old pmap->pm_active */
 	movq	TD_PROC(%rdi), %rdx		/* oldproc */
 	movq	P_VMSPACE(%rdx), %rdx
-	btrq	%rax, VM_PMAP+PM_ACTIVE(%rdx)	/* clear old */
+#ifdef SMP
+	lock
+#endif
+	btrl	%eax, VM_PMAP+PM_ACTIVE(%rdx)	/* clear old */
 
 	/* Set bit in new pmap->pm_active */
 	movq	TD_PROC(%rsi),%rdx		/* newproc */
 	movq	P_VMSPACE(%rdx), %rdx
-	btsq	%rax, VM_PMAP+PM_ACTIVE(%rdx)	/* set new */
+#ifdef SMP
+	lock
+#endif
+	btsl	%eax, VM_PMAP+PM_ACTIVE(%rdx)	/* set new */
 
 sw1:
 	/*
@@ -191,8 +201,11 @@ sw1:
 	wrmsr
 
 	/* Update the TSS_RSP0 pointer for the next interrupt */
+	movq	PCPU(TSSP), %rax
+	addq	$COMMON_TSS_RSP0, %rax
 	leaq	-16(%r8), %rbx
-	movq	%rbx, common_tss + COMMON_TSS_RSP0
+	movq	%rbx, (%rax)
+	movq	%rbx, PCPU(RSP0)
 
 	/* Restore context. */
 	movq	PCB_RBX(%r8),%rbx
diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c
index 3dd6a8a..077c914 100644
--- a/sys/amd64/amd64/db_interface.c
+++ b/sys/amd64/amd64/db_interface.c
@@ -98,6 +98,22 @@ kdb_trap(int type, int code, struct amd64_saved_state *regs)
 	ef = read_rflags();
 	disable_intr();
 
+#ifdef SMP
+
+#if defined(VERBOSE_CPUSTOP_ON_DDBBREAK)
+	db_printf("\nCPU%d stopping CPUs: 0x%08x...", PCPU_GET(cpuid),
+	    PCPU_GET(other_cpus));
+#endif /* VERBOSE_CPUSTOP_ON_DDBBREAK */
+
+	/* We stop all CPUs except ourselves (obviously) */
+	stop_cpus(PCPU_GET(other_cpus));
+
+#if defined(VERBOSE_CPUSTOP_ON_DDBBREAK)
+	db_printf(" stopped.\n");
+#endif /* VERBOSE_CPUSTOP_ON_DDBBREAK */
+
+#endif /* SMP */
+
 	switch (type) {
 	    case T_BPTFLT:	/* breakpoint */
 	    case T_TRCTRAP:	/* debug exception */
@@ -192,6 +208,27 @@ kdb_trap(int type, int code, struct amd64_saved_state *regs)
 	regs->tf_ds     = ddb_regs.tf_ds & 0xffff;
 #endif
 
+#ifdef SMP
+
+#if defined(VERBOSE_CPUSTOP_ON_DDBBREAK)
+	db_printf("\nCPU%d restarting CPUs: 0x%08x...", PCPU_GET(cpuid),
+	    stopped_cpus);
+#endif /* VERBOSE_CPUSTOP_ON_DDBBREAK */
+
+	/* Restart all the CPUs we previously stopped */
+	if (stopped_cpus != PCPU_GET(other_cpus) && smp_started != 0) {
+		db_printf("whoa, other_cpus: 0x%08x, stopped_cpus: 0x%08x\n",
+			  PCPU_GET(other_cpus), stopped_cpus);
+		panic("stop_cpus() failed");
+	}
+	restart_cpus(stopped_cpus);
+
+#if defined(VERBOSE_CPUSTOP_ON_DDBBREAK)
+	db_printf(" restarted.\n");
+#endif /* VERBOSE_CPUSTOP_ON_DDBBREAK */
+
+#endif /* SMP */
+
 	write_rflags(ef);
 
 	return (1);
diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c
index a05348a..7dba9bb 100644
--- a/sys/amd64/amd64/db_trace.c
+++ b/sys/amd64/amd64/db_trace.c
@@ -245,8 +245,9 @@ db_nextframe(fp, ip, p)
 		if (strcmp(name, "calltrap") == 0 ||
 		    strcmp(name, "fork_trampoline") == 0)
 			frame_type = TRAP;
-		else if (strncmp(name, "Xintr", 5) == 0 ||
-		    strncmp(name, "Xfastintr", 9) == 0)
+		else if (strncmp(name, "Xatpic_intr", 11) == 0 ||
+		    strncmp(name, "Xatpic_fastintr", 15) == 0 ||
+		    strncmp(name, "Xapic_isr", 9) == 0)
 			frame_type = INTERRUPT;
 		else if (strcmp(name, "Xfast_syscall") == 0)
 			frame_type = SYSCALL;
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 3d2eaa6..972f19c 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -35,14 +35,11 @@
  */
 
 #include <machine/asmacros.h>
-#include <sys/mutex.h>
 #include <machine/psl.h>
 #include <machine/trap.h>
 
 #include "assym.s"
 
-#define	SEL_RPL_MASK	0x0003
-
 	.text
 
 /*****************************************************************************/
@@ -72,8 +69,6 @@
  * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
  * must load them with appropriate values for supervisor mode operation.
  */
-#define	IDTVEC(name)	ALIGN_TEXT; .globl __CONCAT(X,name); \
-			.type __CONCAT(X,name),@function; __CONCAT(X,name):
 
 MCOUNT_LABEL(user)
 MCOUNT_LABEL(btrap)
@@ -223,7 +218,7 @@ IDTVEC(page)
 IDTVEC(fast_syscall)
 	swapgs
 	movq	%rsp,PCPU(SCRATCH_RSP)
-	movq	common_tss+COMMON_TSS_RSP0,%rsp
+	movq	PCPU(RSP0),%rsp
 	/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
 	subq	$TF_SIZE,%rsp
 	/* defer TF_RSP till we have a spare register */
@@ -297,14 +292,6 @@ ENTRY(fork_trampoline)
 	call	fork_exit
 	jmp	doreti			/* Handle any ASTs */
 
-
-/*
- * Include what was once config+isa-dependent code.
- * XXX it should be in a stand-alone file.  It's still icu-dependent and
- * belongs in i386/isa.
- */
-#include "amd64/isa/vector.S"
-
 	.data
 	ALIGN_DATA
 
@@ -406,5 +393,3 @@ doreti_iret_fault:
 	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
 	movq	$0,TF_ERR(%rsp)	/* XXX should be the error code */
 	jmp	alltraps_with_regs_pushed
-
-#include "amd64/isa/icu_ipl.S"
diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c
index 1e4890c..1acb931 100644
--- a/sys/amd64/amd64/fpu.c
+++ b/sys/amd64/amd64/fpu.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/cputypes.h>
 #include <machine/frame.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
 #include <machine/psl.h>
@@ -63,8 +64,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/segments.h>
 #include <machine/ucontext.h>
 
-#include <amd64/isa/intr_machdep.h>
-
 /*
  * Floating point support.
  */
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 6a017e5..27a1a12 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -69,10 +69,12 @@ __FBSDID("$FreeBSD$");
 #include <nfs/rpcv2.h>
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsdiskless.h>
+#include <machine/apicreg.h>
 #include <machine/cpu.h>
 #include <machine/sigframe.h>
 #include <machine/proc.h>
 #include <machine/specialreg.h>
+#include <machine/segments.h>
 
 ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
@@ -83,11 +85,6 @@ ASSYM(P_UAREA, offsetof(struct proc, p_uarea));
 ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
 ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
 ASSYM(TD_PROC, offsetof(struct thread, td_proc));
-ASSYM(TD_INTR_NESTING_LEVEL, offsetof(struct thread, td_intr_nesting_level));
-ASSYM(TD_CRITNEST, offsetof(struct thread, td_critnest));
-ASSYM(TD_MD, offsetof(struct thread, td_md));
-
-ASSYM(P_MD, offsetof(struct proc, p_md));
 
 ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
 ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
@@ -180,6 +177,7 @@ ASSYM(UC_EFLAGS, offsetof(ucontext_t, uc_mcontext.mc_rflags));
 ASSYM(ENOENT, ENOENT);
 ASSYM(EFAULT, EFAULT);
 ASSYM(ENAMETOOLONG, ENAMETOOLONG);
+ASSYM(MAXCOMLEN, MAXCOMLEN);
 ASSYM(MAXPATHLEN, MAXPATHLEN);
 ASSYM(PC_SIZEOF, sizeof(struct pcpu));
 ASSYM(PC_PRVSPACE, offsetof(struct pcpu, pc_prvspace));
@@ -189,12 +187,24 @@ ASSYM(PC_IDLETHREAD, offsetof(struct pcpu, pc_idlethread));
 ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
 ASSYM(PC_SCRATCH_RSP, offsetof(struct pcpu, pc_scratch_rsp));
+ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
+ASSYM(PC_TSSP, offsetof(struct pcpu, pc_tssp));
+ASSYM(PC_RSP0, offsetof(struct pcpu, pc_rsp0));
+ 
+ASSYM(LA_VER, offsetof(struct LAPIC, version));
+ASSYM(LA_TPR, offsetof(struct LAPIC, tpr));
+ASSYM(LA_EOI, offsetof(struct LAPIC, eoi));
+ASSYM(LA_SVR, offsetof(struct LAPIC, svr));
+ASSYM(LA_ICR_LO, offsetof(struct LAPIC, icr_lo));
+ASSYM(LA_ICR_HI, offsetof(struct LAPIC, icr_hi));
+ASSYM(LA_ISR, offsetof(struct LAPIC, isr0));
 
 ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL));
 ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL));
 ASSYM(KUCSEL, GSEL(GUCODE_SEL, SEL_UPL));
 ASSYM(KUDSEL, GSEL(GUDATA_SEL, SEL_UPL));
 ASSYM(KUC32SEL, GSEL(GUCODE32_SEL, SEL_UPL));
+ASSYM(SEL_RPL_MASK, SEL_RPL_MASK);
 
 ASSYM(MSR_FSBASE, MSR_FSBASE);
 ASSYM(MSR_GSBASE, MSR_GSBASE);
diff --git a/sys/amd64/amd64/identcpu.c b/sys/amd64/amd64/identcpu.c
index ba8e58e..f3d70c2 100644
--- a/sys/amd64/amd64/identcpu.c
+++ b/sys/amd64/amd64/identcpu.c
@@ -55,12 +55,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/asmacros.h>
 #include <machine/clock.h>
 #include <machine/cputypes.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
 #include <machine/segments.h>
 #include <machine/specialreg.h>
 #include <machine/md_var.h>
 
 #include <amd64/isa/icu.h>
-#include <amd64/isa/intr_machdep.h>
 
 /* XXX - should be in header file: */
 void printcpuinfo(void);
diff --git a/sys/amd64/amd64/io_apic.c b/sys/amd64/amd64/io_apic.c
index 4af70fa..b620440 100644
--- a/sys/amd64/amd64/io_apic.c
+++ b/sys/amd64/amd64/io_apic.c
@@ -30,6 +30,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_atpic.h"
 #include "opt_isa.h"
 #include "opt_no_mixed_mode.h"
 
@@ -50,8 +51,8 @@ __FBSDID("$FreeBSD$");
 #include <machine/apicvar.h>
 #include <machine/segments.h>
 
-#if defined(DEV_ISA) && !defined(NO_MIXED_MODE)
-#define MIXED_MODE
+#if defined(DEV_ISA) && defined(DEV_ATPIC) && !defined(NO_MIXED_MODE)
+#define	MIXED_MODE
 #endif
 
 #define IOAPIC_ISA_INTS		16
diff --git a/sys/amd64/amd64/local_apic.c b/sys/amd64/amd64/local_apic.c
index 6f942bf..bdff518 100644
--- a/sys/amd64/amd64/local_apic.c
+++ b/sys/amd64/amd64/local_apic.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/pcpu.h>
+#include <sys/proc.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
@@ -171,8 +172,7 @@ lapic_init(uintptr_t addr)
 	KASSERT(trunc_page(addr) == addr,
 	    ("local APIC not aligned on a page boundary"));
 	lapic = (lapic_t *)pmap_mapdev(addr, sizeof(lapic_t));
-	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYS386IGT, SEL_KPL,
-	    GSEL(GCODE_SEL, SEL_KPL));
+	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
 
 	/* Perform basic initialization of the BSP's local APIC. */
 	value = lapic->svr;
@@ -242,8 +242,7 @@ lapic_enable_intr(u_int irq)
 	KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
 	KASSERT(ioint_handlers[vector / 32] != NULL,
 	    ("No ISR handler for IRQ %u", irq));
-	setidt(vector, ioint_handlers[vector / 32], SDT_SYS386IGT, SEL_KPL,
-	    GSEL(GCODE_SEL, SEL_KPL));
+	setidt(vector, ioint_handlers[vector / 32], SDT_SYSIGT, SEL_KPL,  0);
 }
 
 void
@@ -478,13 +477,14 @@ lapic_eoi(void)
 }
 
 void
-lapic_handle_intr(struct intrframe frame)
+lapic_handle_intr(void *cookie, struct intrframe frame)
 {
 	struct intsrc *isrc;
+	int vec = (uintptr_t)cookie;
 
-	if (frame.if_vec == -1)
+	if (vec == -1)
 		panic("Couldn't get vector from ISR!");
-	isrc = intr_lookup_source(apic_idt_to_irq(frame.if_vec));
+	isrc = intr_lookup_source(apic_idt_to_irq(vec));
 	intr_execute_handlers(isrc, &frame);
 }
 
@@ -589,21 +589,9 @@ static void
 apic_setup_local(void *dummy __unused)
 {
 	int retval;
-	uint64_t apic_base;
 
 	if (best_enum == NULL)
 		return;
-	/*
-	 * To work around an errata, we disable the local APIC on some
-	 * CPUs during early startup.  We need to turn the local APIC back
-	 * on on such CPUs now.
-	 */
-	if (cpu == CPU_686 && strcmp(cpu_vendor, "GenuineIntel") == 0 &&
-	    (cpu_id & 0xff0) == 0x610) {
-		apic_base = rdmsr(MSR_APICBASE);
-		apic_base |= APICBASE_ENABLED;
-		wrmsr(MSR_APICBASE, apic_base);
-	}
 	retval = best_enum->apic_setup_local();
 	if (retval != 0)
 		printf("%s: Failed to setup the local APIC: returned %d\n",
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index e32d82a..2140d7a 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -42,6 +42,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_atalk.h"
+#include "opt_atpic.h"
 #include "opt_compat.h"
 #include "opt_cpu.h"
 #include "opt_ddb.h"
@@ -101,6 +102,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/reg.h>
 #include <machine/clock.h>
 #include <machine/specialreg.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/metadata.h>
 #include <machine/proc.h>
@@ -108,9 +110,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/perfmon.h>
 #endif
 #include <machine/tss.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 
 #include <amd64/isa/icu.h>
-#include <amd64/isa/intr_machdep.h>
+
+#include <isa/isareg.h>
 #include <isa/rtc.h>
 #include <sys/ptrace.h>
 #include <machine/sigframe.h>
@@ -146,7 +152,9 @@ vm_paddr_t phys_avail[10];
 struct kva_md_info kmi;
 
 static struct trapframe proc0_tf;
-static struct pcpu __pcpu;
+struct region_descriptor r_gdt, r_idt;
+
+struct pcpu __pcpu[MAXCPU];
 
 struct mtx icu_lock;
 
@@ -196,7 +204,6 @@ cpu_startup(dummy)
 	bufinit();
 	vm_pager_bufferinit();
 
-	/* For SMP, we delay the cpu_setregs() until after SMP startup. */
 	cpu_setregs();
 }
 
@@ -589,13 +596,13 @@ SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock,
  * Initialize segments & interrupt table
  */
 
-struct user_segment_descriptor gdt[NGDT];/* global descriptor table */
+struct user_segment_descriptor gdt[NGDT * MAXCPU];/* global descriptor table */
 static struct gate_descriptor idt0[NIDT];
 struct gate_descriptor *idt = &idt0[0];	/* interrupt descriptor table */
 
 static char dblfault_stack[PAGE_SIZE] __aligned(16);
 
-struct amd64tss common_tss;
+struct amd64tss common_tss[MAXCPU];
 
 /* software prototypes -- in more palatable form */
 struct soft_segment_descriptor gdt_segs[] = {
@@ -755,6 +762,15 @@ ssdtosyssd(ssd, sd)
 	sd->sd_gran  = ssd->ssd_gran;
 }
 
+#if !defined(DEV_ATPIC) && defined(DEV_ISA)
+#include <isa/isavar.h>
+u_int
+isa_irq_pending(void)
+{
+
+	return (0);
+}
+#endif
 
 #define PHYSMAP_SIZE	(2 * 8)
 
@@ -783,7 +799,6 @@ static void
 getmemsize(caddr_t kmdp, u_int64_t first)
 {
 	int i, physmap_idx, pa_indx;
-	u_int extmem;
 	vm_paddr_t pa, physmap[PHYSMAP_SIZE];
 	pt_entry_t *pte;
 	char *cp;
@@ -802,12 +817,9 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 	 * ie: an int32_t immediately precedes smap.
 	 */
 	smapbase = (struct bios_smap *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_SMAP);
-	if (smapbase == 0)
-		smapbase = (struct bios_smap *)preload_search_info(kmdp, MODINFO_METADATA | 0x0009);	/* Old value for MODINFOMD_SMAP */
-	if (smapbase == 0) {
+	if (smapbase == NULL)
 		panic("No BIOS smap info from loader!");
-		goto deep_shit;
-	}
+
 	smapsize = *((u_int32_t *)smapbase - 1);
 	smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
 
@@ -816,14 +828,11 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 			printf("SMAP type=%02x base=%016lx len=%016lx\n",
 			    smap->type, smap->base, smap->length);
 
-		if (smap->type != 0x01) {
+		if (smap->type != 0x01)
 			continue;
-		}
 
-		if (smap->length == 0) {
-next_run:
+		if (smap->length == 0)
 			continue;
-		}
 
 		for (i = 0; i <= physmap_idx; i += 2) {
 			if (smap->base < physmap[i + 1]) {
@@ -836,6 +845,7 @@ next_run:
 
 		if (smap->base == physmap[physmap_idx + 1]) {
 			physmap[physmap_idx + 1] += smap->length;
+next_run:
 			continue;
 		}
 
@@ -850,69 +860,23 @@ next_run:
 	}
 
 	/*
-	 * Perform "base memory" related probes & setup based on SMAP
+	 * Find the 'base memory' segment for SMP
 	 */
-deep_shit:
-	if (basemem == 0) {
-		for (i = 0; i <= physmap_idx; i += 2) {
-			if (physmap[i] == 0x00000000) {
-				basemem = physmap[i + 1] / 1024;
-				break;
-			}
-		}
-
-		if (basemem == 0) {
-			basemem = rtcin(RTC_BASELO) + (rtcin(RTC_BASEHI) << 8);
-		}
-
-		if (basemem == 0) {
-			basemem = 640;
-		}
-
-		if (basemem > 640) {
-			printf("Preposterous BIOS basemem of %uK, truncating to 640K\n",
-				basemem);
-			basemem = 640;
+	basemem = 0;
+	for (i = 0; i <= physmap_idx; i += 2) {
+		if (physmap[i] == 0x00000000) {
+			basemem = physmap[i + 1] / 1024;
+			break;
 		}
-
-#if 0
-		for (pa = trunc_page(basemem * 1024);
-		     pa < ISA_HOLE_START; pa += PAGE_SIZE)
-			pmap_kenter(KERNBASE + pa, pa);
-#endif
 	}
+	if (basemem == 0)
+		panic("BIOS smap did not include a basemem segment!");
 
-	if (physmap[1] != 0)
-		goto physmap_done;
-
-	/*
-	 * Prefer the RTC value for extended memory.
-	 */
-	extmem = rtcin(RTC_EXTLO) + (rtcin(RTC_EXTHI) << 8);
-
-	/*
-	 * Special hack for chipsets that still remap the 384k hole when
-	 * there's 16MB of memory - this really confuses people that
-	 * are trying to use bus mastering ISA controllers with the
-	 * "16MB limit"; they only have 16MB, but the remapping puts
-	 * them beyond the limit.
-	 *
-	 * If extended memory is between 15-16MB (16-17MB phys address range),
-	 *	chop it to 15MB.
-	 */
-	if ((extmem > 15 * 1024) && (extmem < 16 * 1024))
-		extmem = 15 * 1024;
-
-	physmap[0] = 0;
-	physmap[1] = basemem * 1024;
-	physmap_idx = 2;
-	physmap[physmap_idx] = 0x100000;
-	physmap[physmap_idx + 1] = physmap[physmap_idx] + extmem * 1024;
+#ifdef SMP
+	/* make hole for AP bootstrap code */
+	physmap[1] = mp_bootaddress(physmap[1] / 1024);
+#endif
 
-physmap_done:
-	/*
-	 * Now, physmap contains a map of physical memory.
-	 */
 	/*
 	 * Maxmem isn't the "maximum memory", it's one larger than the
 	 * highest page of the physical address space.  It should be
@@ -929,7 +893,8 @@ physmap_done:
 	 * hw.physmem is a size in bytes; we also allow k, m, and g suffixes
 	 * for the appropriate modifiers.  This overrides MAXMEM.
 	 */
-	if ((cp = getenv("hw.physmem")) != NULL) {
+	cp = getenv("hw.physmem");
+	if (cp != NULL) {
 		u_int64_t AllowMem, sanity;
 		char *ep;
 
@@ -1106,11 +1071,18 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 {
 	caddr_t kmdp;
 	int gsel_tss, off, x;
-	struct region_descriptor r_gdt, r_idt;
 	struct pcpu *pc;
 	u_int64_t msr;
 	char *env;
 
+#ifdef DEV_ISA
+	/* Preemptively mask the atpics and leave them shut down */
+	outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff);
+	outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff);
+#else
+#error "have you forgotten the isa device?";
+#endif
+
 	/* Turn on PTE NX (no execute) bit */
 	msr = rdmsr(MSR_EFER) | EFER_NXE;
 	wrmsr(MSR_EFER, msr);
@@ -1146,7 +1118,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	/*
 	 * make gdt memory segments
 	 */
-	gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss;
+	gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss[0];
 
 	for (x = 0; x < NGDT; x++) {
 		if (x != GPROC0_SEL && x != (GPROC0_SEL + 1))
@@ -1157,7 +1129,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
 	r_gdt.rd_base =  (long) gdt;
 	lgdt(&r_gdt);
-	pc = &__pcpu;
+	pc = &__pcpu[0];
 
 	wrmsr(MSR_FSBASE, 0);		/* User value */
 	wrmsr(MSR_GSBASE, (u_int64_t)pc);
@@ -1166,6 +1138,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	pcpu_init(pc, 0, sizeof(struct pcpu));
 	PCPU_SET(prvspace, pc);
 	PCPU_SET(curthread, &thread0);
+	PCPU_SET(tssp, &common_tss[0]);
 
 	/*
 	 * Initialize mutexes.
@@ -1211,8 +1184,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	 */
 	cninit();
 
-#ifdef DEV_ISA
-	isa_defaultirq();
+#ifdef DEV_ATPIC
+	atpic_startup();
 #endif
 
 #ifdef DDB
@@ -1225,12 +1198,14 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	initializecpu();	/* Initialize CPU registers */
 
 	/* make an initial tss so cpu can get interrupt stack on syscall! */
-	common_tss.tss_rsp0 = thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb);
+	common_tss[0].tss_rsp0 = thread0.td_kstack + \
+	    KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb);
 	/* Ensure the stack is aligned to 16 bytes */
-	common_tss.tss_rsp0 &= ~0xF;
+	common_tss[0].tss_rsp0 &= ~0xF;
+	PCPU_SET(rsp0, common_tss[0].tss_rsp0);
 
 	/* doublefault stack space, runs on ist1 */
-	common_tss.tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)];
+	common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)];
 
 	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
 	ltr(gsel_tss);
diff --git a/sys/amd64/amd64/mem.c b/sys/amd64/amd64/mem.c
index 56d268c..aeb2e35 100644
--- a/sys/amd64/amd64/mem.c
+++ b/sys/amd64/amd64/mem.c
@@ -323,6 +323,15 @@ mem_range_attr_set(struct mem_range_desc *mrd, int *arg)
 	return (mem_range_softc.mr_op->set(&mem_range_softc, mrd, arg));
 }
 
+#ifdef SMP
+void
+mem_range_AP_init(void)
+{
+	if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP)
+		(mem_range_softc.mr_op->initAP(&mem_range_softc));
+}
+#endif
+
 static int
 mem_modevent(module_t mod, int type, void *data)
 {
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index f58a94f..fcd478b 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 1996, by Steve Passe
+ * Copyright (c) 2003, by Peter Wemm
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,30 +27,12 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_apic.h"
 #include "opt_cpu.h"
 #include "opt_kstack_pages.h"
 
-#if !defined(lint)
-#if !defined(SMP)
-#error How did you get here?
-#endif
-
-#if defined(I386_CPU) && !defined(COMPILING_LINT)
-#error SMP not supported with I386_CPU
-#endif
-#ifndef DEV_APIC
-#error The apic device is required for SMP, add "device apic" to your config file.
-#endif
-#if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT)
-#error SMP not supported with CPU_DISABLE_CMPXCHG
-#endif
-#endif /* not lint */
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
-#include <sys/cons.h>	/* cngetc() */
 #ifdef GPROF 
 #include <sys/gmon.h>
 #endif
@@ -75,9 +58,8 @@ __FBSDID("$FreeBSD$");
 #include <machine/md_var.h>
 #include <machine/pcb.h>
 #include <machine/smp.h>
-#include <machine/smptests.h>	/** COUNT_XINVLTLB_HITS */
 #include <machine/specialreg.h>
-#include <machine/privatespace.h>
+#include <machine/tss.h>
 
 #define WARMBOOT_TARGET		0
 #define WARMBOOT_OFF		(KERNBASE + 0x0467)
@@ -88,67 +70,9 @@ __FBSDID("$FreeBSD$");
 #define BIOS_RESET		(0x0f)
 #define BIOS_WARM		(0x0a)
 
-/*
- * this code MUST be enabled here and in mpboot.s.
- * it follows the very early stages of AP boot by placing values in CMOS ram.
- * it NORMALLY will never be needed and thus the primitive method for enabling.
- *
-#define CHECK_POINTS
- */
-
-#if defined(CHECK_POINTS) && !defined(PC98)
-#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
-#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
-
-#define CHECK_INIT(D);				\
-	CHECK_WRITE(0x34, (D));			\
-	CHECK_WRITE(0x35, (D));			\
-	CHECK_WRITE(0x36, (D));			\
-	CHECK_WRITE(0x37, (D));			\
-	CHECK_WRITE(0x38, (D));			\
-	CHECK_WRITE(0x39, (D));
-
-#define CHECK_PRINT(S);				\
-	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
-	   (S),					\
-	   CHECK_READ(0x34),			\
-	   CHECK_READ(0x35),			\
-	   CHECK_READ(0x36),			\
-	   CHECK_READ(0x37),			\
-	   CHECK_READ(0x38),			\
-	   CHECK_READ(0x39));
-
-#else				/* CHECK_POINTS */
-
-#define CHECK_INIT(D)
-#define CHECK_PRINT(S)
-#define CHECK_WRITE(A, D)
-
-#endif				/* CHECK_POINTS */
-
-/*
- * Values to send to the POST hardware.
- */
-#define MP_BOOTADDRESS_POST	0x10
-#define MP_PROBE_POST		0x11
-#define MPTABLE_PASS1_POST	0x12
-
-#define MP_START_POST		0x13
-#define MP_ENABLE_POST		0x14
-#define MPTABLE_PASS2_POST	0x15
-
-#define START_ALL_APS_POST	0x16
-#define INSTALL_AP_TRAMP_POST	0x17
-#define START_AP_POST		0x18
-
-#define MP_ANNOUNCE_POST	0x19
-
 /* lock region used by kernel profiling */
 int	mcount_lock;
 
-/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
-int	current_postcode;
-
 int	mp_naps;		/* # of Applications processors */
 int	boot_cpu_id = -1;	/* designated BSP */
 extern	int nkpt;
@@ -164,6 +88,9 @@ struct cpu_top *smp_topology;
 char *bootSTK;
 static int bootAP;
 
+/* Free these after use */
+void *bootstacks[MAXCPU];
+
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
 
@@ -178,6 +105,8 @@ vm_offset_t smp_tlb_addr2;
 volatile int smp_tlb_wait;
 struct mtx smp_tlb_mtx;
 
+extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
+
 /*
  * Local data and functions.
  */
@@ -201,17 +130,17 @@ struct cpu_info {
 } static cpu_info[MAXCPU];
 static int cpu_apic_ids[MAXCPU];
 
-static u_int boot_address;
+static u_int	boot_address;
 
 static void	set_logical_apic_ids(void);
 static int	start_all_aps(void);
-static void	install_ap_tramp(void);
 static int	start_ap(int apic_id);
 static void	release_aps(void *dummy);
 
 static int	hlt_cpus_mask;
 static int	hlt_logical_cpus;
 static struct	sysctl_ctx_list logical_cpu_clist;
+static u_int	bootMP_size;
 
 /*
  * Calculate usable address in base memory for AP trampoline code.
@@ -219,13 +148,15 @@ static struct	sysctl_ctx_list logical_cpu_clist;
 u_int
 mp_bootaddress(u_int basemem)
 {
-	POSTCODE(MP_BOOTADDRESS_POST);
 
-	boot_address = trunc_page(basemem);	/* round down to 4k boundary */
+	bootMP_size = mptramp_end - mptramp_start;
+	boot_address = trunc_page(basemem * 1024); /* round down to 4k boundary */
 	if ((basemem - boot_address) < bootMP_size)
 		boot_address -= PAGE_SIZE;	/* not enough, lower by 4k */
+	/* 3 levels of page table pages */
+	mptramp_pagetables = boot_address - (PAGE_SIZE * 3);
 
-	return boot_address;
+	return mptramp_pagetables;
 }
 
 void
@@ -302,43 +233,34 @@ cpu_mp_start(void)
 {
 	int i;
 
-	POSTCODE(MP_START_POST);
-
 	/* Initialize the logical ID to APIC ID table. */
 	for (i = 0; i < MAXCPU; i++)
 		cpu_apic_ids[i] = -1;
 
 	/* Install an inter-CPU IPI for TLB invalidation */
-	setidt(IPI_INVLTLB, IDTVEC(invltlb),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	setidt(IPI_INVLPG, IDTVEC(invlpg),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	setidt(IPI_INVLRNG, IDTVEC(invlrng),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0);
+	setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0);
+	setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0);
 
 	/* Install an inter-CPU IPI for forwarding hardclock() */
-	setidt(IPI_HARDCLOCK, IDTVEC(hardclock),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_HARDCLOCK, IDTVEC(hardclock), SDT_SYSIGT, SEL_KPL, 0);
 	
 	/* Install an inter-CPU IPI for forwarding statclock() */
-	setidt(IPI_STATCLOCK, IDTVEC(statclock),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_STATCLOCK, IDTVEC(statclock), SDT_SYSIGT, SEL_KPL, 0);
 	
+#ifdef LAZY_SWITCH
 	/* Install an inter-CPU IPI for lazy pmap release */
-	setidt(IPI_LAZYPMAP, IDTVEC(lazypmap),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_LAZYPMAP, IDTVEC(lazypmap), SDT_SYSIGT, SEL_KPL, 0);
+#endif
 
 	/* Install an inter-CPU IPI for all-CPU rendezvous */
-	setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);
 
 	/* Install an inter-CPU IPI for forcing an additional software trap */
-	setidt(IPI_AST, IDTVEC(cpuast),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_AST, IDTVEC(cpuast), SDT_SYSIGT, SEL_KPL, 0);
 
 	/* Install an inter-CPU IPI for CPU stop/restart */
-	setidt(IPI_STOP, IDTVEC(cpustop),
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0);
 
 	mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN);
 
@@ -371,8 +293,6 @@ cpu_mp_announce(void)
 {
 	int i, x;
 
-	POSTCODE(MP_ANNOUNCE_POST);
-
 	/* List CPUs */
 	printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id);
 	for (i = 1, x = 0; x < MAXCPU; x++) {
@@ -390,38 +310,41 @@ cpu_mp_announce(void)
 void
 init_secondary(void)
 {
-	int	gsel_tss;
-	int	x, myid;
-	u_int	cr0;
-
-	/* bootAP is set in start_ap() to our ID. */
-	myid = bootAP;
-	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
-	gdt_segs[GPROC0_SEL].ssd_base =
-		(int) &SMP_prvspace[myid].pcpu.pc_common_tss;
-	SMP_prvspace[myid].pcpu.pc_prvspace =
-		&SMP_prvspace[myid].pcpu;
-
-	for (x = 0; x < NGDT; x++) {
-		ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd);
-	}
+	struct pcpu *pc;
+	u_int64_t msr, cr0;
+	int cpu, gsel_tss;
+
+	/* Set by the startup code for us to use */
+	cpu = bootAP;
+
+	/* Init tss */
+	common_tss[cpu] = common_tss[0];
+	common_tss[cpu].tss_rsp0 = 0;   /* not used until after switch */
+
+	gdt_segs[GPROC0_SEL].ssd_base = (long) &common_tss[cpu];
+	ssdtosyssd(&gdt_segs[GPROC0_SEL],
+	   (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
 
-	r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
-	r_gdt.rd_base = (int) &gdt[myid * NGDT];
 	lgdt(&r_gdt);			/* does magic intra-segment return */
 
-	lidt(&r_idt);
+	/* Get per-cpu data */
+	pc = &__pcpu[cpu];
+
+	/* prime data page for it to use */
+	pcpu_init(pc, cpu, sizeof(struct pcpu));
+	pc->pc_apic_id = cpu_apic_ids[cpu];
+	pc->pc_prvspace = pc;
+	pc->pc_curthread = 0;
+	pc->pc_tssp = &common_tss[cpu];
+	pc->pc_rsp0 = 0;
 
-	lldt(_default_ldt);
-	PCPU_SET(currentldt, _default_ldt);
+	wrmsr(MSR_FSBASE, 0);		/* User value */
+	wrmsr(MSR_GSBASE, (u_int64_t)pc);
+	wrmsr(MSR_KGSBASE, (u_int64_t)pc);	/* XXX User value while we're in the kernel */
+
+	lidt(&r_idt);
 
 	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
-	gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
-	PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */
-	PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL));
-	PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16);
-	PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd);
-	PCPU_SET(common_tssd, *PCPU_GET(tss_gdt));
 	ltr(gsel_tss);
 
 	/*
@@ -432,32 +355,32 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-	CHECK_WRITE(0x38, 5);
-	
-	/* Disable local APIC just to be sure. */
+
+	/* Set up the fast syscall stuff */
+	msr = rdmsr(MSR_EFER) | EFER_SCE;
+	wrmsr(MSR_EFER, msr);
+	wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall));
+	wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+	msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
+	      ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
+	wrmsr(MSR_STAR, msr);
+	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
+
+	/* Disable local apic just to be sure. */
 	lapic_disable();
 
 	/* signal our startup to the BSP. */
 	mp_naps++;
-	CHECK_WRITE(0x39, 6);
 
 	/* Spin until the BSP releases the AP's. */
 	while (!aps_ready)
 		ia32_pause();
 
-	/* BSP may have changed PTD while we were waiting */
-	invltlb();
-	pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
-
-#if defined(I586_CPU) && !defined(NO_F00F_HACK)
-	lidt(&r_idt);
-#endif
-
 	/* set up CPU registers and state */
 	cpu_setregs();
 
 	/* set up FPU state on the AP */
-	npxinit(__INITIAL_NPXCW__);
+	fpuinit();
 
 	/* set up SSE registers */
 	enable_sse();
@@ -467,7 +390,6 @@ init_secondary(void)
 		printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
 		printf("SMP: actual apic_id = %d\n", lapic_id());
 		printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
-		printf("PTD[MPPTDI] = %#jx\n", (uintmax_t)PTD[MPPTDI]);
 		panic("cpuid mismatch! boom!!");
 	}
 
@@ -559,39 +481,51 @@ set_logical_apic_ids(void)
 static int
 start_all_aps(void)
 {
-#ifndef PC98
 	u_char mpbiosreason;
-#endif
-	u_long mpbioswarmvec;
-	struct pcpu *pc;
-	char *stack;
-	uintptr_t kptbase;
-	int i, pg, apic_id, cpu;
-
-	POSTCODE(START_ALL_APS_POST);
+	u_int32_t mpbioswarmvec;
+	int apic_id, cpu, i;
+	u_int64_t *pt4, *pt3, *pt2;
 
 	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
 
 	/* install the AP 1st level boot code */
-	install_ap_tramp();
+	pmap_kenter(boot_address + KERNBASE, boot_address);
+	bcopy(mptramp_start, (void *)((uintptr_t)boot_address + KERNBASE), bootMP_size);
+
+	/* Locate the page tables, they'll be below the trampoline */
+	pt4 = (u_int64_t *)(uintptr_t)(mptramp_pagetables + KERNBASE);
+	pt3 = pt4 + (PAGE_SIZE) / sizeof(u_int64_t);
+	pt2 = pt3 + (PAGE_SIZE) / sizeof(u_int64_t);
+
+	/* Create the initial 1GB replicated page tables */
+	for (i = 0; i < 512; i++) {
+		/* Each slot of the level 4 pages points to the same level 3 page */
+		pt4[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + PAGE_SIZE);
+		pt4[i] |= PG_V | PG_RW | PG_U;
+
+		/* Each slot of the level 3 pages points to the same level 2 page */
+		pt3[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + (2 * PAGE_SIZE));
+		pt3[i] |= PG_V | PG_RW | PG_U;
+
+		/* The level 2 page slots are mapped with 2MB pages for 1GB. */
+		pt2[i] = i * (2 * 1024 * 1024);
+		pt2[i] |= PG_V | PG_RW | PG_PS | PG_U;
+	}
 
 	/* save the current value of the warm-start vector */
-	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
-#ifndef PC98
+	mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF);
 	outb(CMOS_REG, BIOS_RESET);
 	mpbiosreason = inb(CMOS_DATA);
-#endif
 
-	/* set up temporary P==V mapping for AP boot */
-	/* XXX this is a hack, we should boot the AP on its own stack/PTD */
-	kptbase = (uintptr_t)(void *)KPTphys;
-	for (i = 0; i < NKPT; i++)
-		PTD[i] = (pd_entry_t)(PG_V | PG_RW |
-		    ((kptbase + i * PAGE_SIZE) & PG_FRAME));
-	invltlb();
+	/* setup a vector to our boot code */
+	*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
+	*((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
+	outb(CMOS_REG, BIOS_RESET);
+	outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
 
 	/* start each AP */
-	for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) {
+	cpu = 0;
+	for (apic_id = 0; apic_id < MAXCPU; apic_id++) {
 		if (!cpu_info[apic_id].cpu_present ||
 		    cpu_info[apic_id].cpu_bsp)
 			continue;
@@ -600,48 +534,18 @@ start_all_aps(void)
 		/* save APIC ID for this logical ID */
 		cpu_apic_ids[cpu] = apic_id;
 
-		/* first page of AP's private space */
-		pg = cpu * i386_btop(sizeof(struct privatespace));
-
-		/* allocate a new private data page */
-		pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE);
-
-		/* wire it into the private page table page */
-		SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(pc));
-
 		/* allocate and set up an idle stack data page */
-		stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */
-		for (i = 0; i < KSTACK_PAGES; i++)
-			SMPpt[pg + 1 + i] = (pt_entry_t)
-			    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
-
-		/* prime data page for it to use */
-		pcpu_init(pc, cpu, sizeof(struct pcpu));
-		pc->pc_apic_id = apic_id;
-
-		/* setup a vector to our boot code */
-		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
-		*((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4);
-#ifndef PC98
-		outb(CMOS_REG, BIOS_RESET);
-		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
-#endif
+		bootstacks[cpu] = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
 
-		bootSTK = &SMP_prvspace[cpu].idlekstack[KSTACK_PAGES *
-		    PAGE_SIZE];
+		bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8;
 		bootAP = cpu;
 
 		/* attempt to start the Application Processor */
-		CHECK_INIT(99);	/* setup checkpoints */
 		if (!start_ap(apic_id)) {
-			printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id);
-			CHECK_PRINT("trace");	/* show checkpoints */
-			/* better panic as the AP may be running loose */
-			printf("panic y/n? [y] ");
-			if (cngetc() != 'n')
-				panic("bye-bye");
+			/* restore the warmstart vector */
+			*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
+			panic("AP #%d (PHY# %d) failed!", cpu, apic_id);
 		}
-		CHECK_PRINT("trace");		/* show checkpoints */
 
 		all_cpus |= (1 << cpu);		/* record AP in CPU map */
 	}
@@ -650,92 +554,15 @@ start_all_aps(void)
 	PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
 
 	/* restore the warmstart vector */
-	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
-#ifndef PC98
+	*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
+
 	outb(CMOS_REG, BIOS_RESET);
 	outb(CMOS_DATA, mpbiosreason);
-#endif
-
-	/*
-	 * Set up the idle context for the BSP.  Similar to above except
-	 * that some was done by locore, some by pmap.c and some is implicit
-	 * because the BSP is cpu#0 and the page is initially zero and also
-	 * because we can refer to variables by name on the BSP..
-	 */
-
-	/* Allocate and setup BSP idle stack */
-	stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE);
-	for (i = 0; i < KSTACK_PAGES; i++)
-		SMPpt[1 + i] = (pt_entry_t)
-		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
-
-	for (i = 0; i < NKPT; i++)
-		PTD[i] = 0;
-	pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1);
 
 	/* number of APs actually started */
 	return mp_naps;
 }
 
-/*
- * load the 1st level AP boot code into base memory.
- */
-
-/* targets for relocation */
-extern void bigJump(void);
-extern void bootCodeSeg(void);
-extern void bootDataSeg(void);
-extern void MPentry(void);
-extern u_int MP_GDT;
-extern u_int mp_gdtbase;
-
-static void
-install_ap_tramp(void)
-{
-	int     x;
-	int     size = *(int *) ((u_long) & bootMP_size);
-	u_char *src = (u_char *) ((u_long) bootMP);
-	u_char *dst = (u_char *) boot_address + KERNBASE;
-	u_int   boot_base = (u_int) bootMP;
-	u_int8_t *dst8;
-	u_int16_t *dst16;
-	u_int32_t *dst32;
-
-	POSTCODE(INSTALL_AP_TRAMP_POST);
-
-	pmap_kenter(boot_address + KERNBASE, boot_address);
-	for (x = 0; x < size; ++x)
-		*dst++ = *src++;
-
-	/*
-	 * modify addresses in code we just moved to basemem. unfortunately we
-	 * need fairly detailed info about mpboot.s for this to work.  changes
-	 * to mpboot.s might require changes here.
-	 */
-
-	/* boot code is located in KERNEL space */
-	dst = (u_char *) boot_address + KERNBASE;
-
-	/* modify the lgdt arg */
-	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
-	*dst32 = boot_address + ((u_int) & MP_GDT - boot_base);
-
-	/* modify the ljmp target for MPentry() */
-	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
-	*dst32 = ((u_int) MPentry - KERNBASE);
-
-	/* modify the target for boot code segment */
-	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
-	dst8 = (u_int8_t *) (dst16 + 1);
-	*dst16 = (u_int) boot_address & 0xffff;
-	*dst8 = ((u_int) boot_address >> 16) & 0xff;
-
-	/* modify the target for boot data segment */
-	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
-	dst8 = (u_int8_t *) (dst16 + 1);
-	*dst16 = (u_int) boot_address & 0xffff;
-	*dst8 = ((u_int) boot_address >> 16) & 0xff;
-}
 
 /*
  * This function starts the AP (application processor) identified
@@ -750,8 +577,6 @@ start_ap(int apic_id)
 	int vector, ms;
 	int cpus;
 
-	POSTCODE(START_AP_POST);
-
 	/* calculate the vector */
 	vector = (boot_address >> 12) & 0xff;
 
@@ -810,50 +635,14 @@ start_ap(int apic_id)
 	DELAY(200);		/* wait ~200uS */
 
 	/* Wait up to 5 seconds for it to start. */
-	for (ms = 0; ms < 5000; ms++) {
+	for (ms = 0; ms < 50; ms++) {
 		if (mp_naps > cpus)
 			return 1;	/* return SUCCESS */
-		DELAY(1000);
+		DELAY(100000);
 	}
 	return 0;		/* return FAILURE */
 }
 
-#ifdef COUNT_XINVLTLB_HITS
-u_int xhits_gbl[MAXCPU];
-u_int xhits_pg[MAXCPU];
-u_int xhits_rng[MAXCPU];
-SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
-SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
-    sizeof(xhits_gbl), "IU", "");
-SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
-    sizeof(xhits_pg), "IU", "");
-SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng,
-    sizeof(xhits_rng), "IU", "");
-
-u_int ipi_global;
-u_int ipi_page;
-u_int ipi_range;
-u_int ipi_range_size;
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
-    0, "");
-
-u_int ipi_masked_global;
-u_int ipi_masked_page;
-u_int ipi_masked_range;
-u_int ipi_masked_range_size;
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW,
-    &ipi_masked_global, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW,
-    &ipi_masked_page, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW,
-    &ipi_masked_range, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW,
-    &ipi_masked_range_size, 0, "");
-#endif /* COUNT_XINVLTLB_HITS */
-
 /*
  * Flush the TLB on all other CPU's
  */
@@ -966,69 +755,49 @@ smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offse
 void
 smp_invltlb(void)
 {
-	if (smp_started) {
+
+	if (smp_started)
 		smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_global++;
-#endif
-	}
 }
 
 void
 smp_invlpg(vm_offset_t addr)
 {
-	if (smp_started) {
+
+	if (smp_started)
 		smp_tlb_shootdown(IPI_INVLPG, addr, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_page++;
-#endif
-	}
 }
 
 void
 smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
 {
-	if (smp_started) {
+
+	if (smp_started)
 		smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_range++;
-		ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
-#endif
-	}
 }
 
 void
 smp_masked_invltlb(u_int mask)
 {
-	if (smp_started) {
+
+	if (smp_started)
 		smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_masked_global++;
-#endif
-	}
 }
 
 void
 smp_masked_invlpg(u_int mask, vm_offset_t addr)
 {
-	if (smp_started) {
+
+	if (smp_started)
 		smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_masked_page++;
-#endif
-	}
 }
 
 void
 smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2)
 {
-	if (smp_started) {
+
+	if (smp_started)
 		smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_masked_range++;
-		ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE;
-#endif
-	}
 }
 
 
diff --git a/sys/amd64/amd64/mpboot.S b/sys/amd64/amd64/mpboot.S
index 8f42f6b..ca53a87 100644
--- a/sys/amd64/amd64/mpboot.S
+++ b/sys/amd64/amd64/mpboot.S
@@ -1,5 +1,5 @@
-/*
- * Copyright (c) 1995, Jack F. Vogel
+/*-
+ * Copyright (c) 2003 Peter Wemm
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -10,16 +10,11 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by Jack F. Vogel
- * 4. The name of the developer may be used to endorse or promote products
- *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -28,257 +23,214 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * mpboot.s:	FreeBSD machine support for the Intel MP Spec
- *		multiprocessor systems.
- *
  * $FreeBSD$
  */
 
-#include "opt_pmap.h"
-
 #include <machine/asmacros.h>		/* miscellaneous asm macros */
-#include <machine/apicreg.h>
 #include <machine/specialreg.h>
 
 #include "assym.s"
 
-#define	R(x)	((x)-KERNBASE)
-
-/*
- * this code MUST be enabled here and in mp_machdep.c
- * it follows the very early stages of AP boot by placing values in CMOS ram.
- * it NORMALLY will never be needed and thus the primitive method for enabling.
- *
-#define CHECK_POINTS
- */
-
-#if defined(CHECK_POINTS) && !defined(PC98)
-
-#define CMOS_REG	(0x70)
-#define CMOS_DATA	(0x71)
-
-#define CHECKPOINT(A,D)		\
-	movb	$(A),%al ;	\
-	outb	%al,$CMOS_REG ;	\
-	movb	$(D),%al ;	\
-	outb	%al,$CMOS_DATA
+	.data				/* So we can modify it */
 
-#else
-
-#define CHECKPOINT(A,D)
-
-#endif /* CHECK_POINTS */
-
-
-/*
- * the APs enter here from their trampoline code (bootMP, below)
- */
-	.p2align 4
-
-NON_GPROF_ENTRY(MPentry)
-	CHECKPOINT(0x36, 3)
+	.p2align 4,0
+	.globl	mptramp_start
+mptramp_start:
+	.code16
 	/*
-	 * Enable features on this processor.  We don't support SMP on
-	 * CPUs older than a Pentium, so we know that we can use the cpuid
-	 * instruction.
+	 * The AP enters here in response to the startup IPI.
+	 * We are in real mode. %cs is the only segment register set.
 	 */
-	movl	$1,%eax
-	cpuid					/* Retrieve features */
-	movl	%cr4,%eax
-#ifndef DISABLE_PSE
-	testl	$CPUID_PSE,%edx
-	jz 1f
-	orl	$CR4_PSE,%eax			/* Enable PSE  */
-1:
-#endif
-#ifndef DISABLE_PG_G
-	testl	$CPUID_PGE,%edx
-	jz 1f
-	orl	$CR4_PGE,%eax			/* Enable PGE  */
-1:	
-#endif
-	testl	$CPUID_VME,%edx
-	jz 1f
-	orl	$CR4_VME,%eax			/* Enable VME  */
-1:
-	movl	%eax,%cr4
-
-	/* Now enable paging mode */
-#ifdef PAE
-	movl	R(IdlePDPT), %eax
-	movl	%eax, %cr3
-	movl	%cr4, %eax
-	orl	$CR4_PAE, %eax
-	movl	%eax, %cr4
-#else
-	movl	R(IdlePTD), %eax
-	movl	%eax,%cr3	
-#endif
-	movl	%cr0,%eax
-	orl	$CR0_PE|CR0_PG,%eax		/* enable paging */
-	movl	%eax,%cr0			/* let the games begin! */
-	movl	bootSTK,%esp			/* boot stack end loc. */
-
-	pushl	$mp_begin			/* jump to high mem */
-	ret
+	cli				/* make sure no interrupts */
+	mov	%cs, %ax		/* copy %cs to %ds.  Remember these */
+	mov	%ax, %ds		/* are offsets rather than selectors */
+	mov	%ax, %ss
 
 	/*
-	 * Wait for the booting CPU to signal startup
+	 * Find relocation base and patch the gdt descript and ljmp targets
 	 */
-mp_begin:	/* now running relocated at KERNBASE */
-	CHECKPOINT(0x37, 4)
-	call	init_secondary			/* load i386 tables */
-
-/*
- * This is the embedded trampoline or bootstrap that is
- * copied into 'real-mode' low memory, it is where the
- * secondary processor "wakes up". When it is executed
- * the processor will eventually jump into the routine
- * MPentry, which resides in normal kernel text above
- * 1Meg.		-jackv
- */
+	xorl	%ebx,%ebx
+	mov	%cs, %bx
+	sall	$4, %ebx		/* %ebx is now our relocation base */
+	orl	%ebx, lgdt_desc-mptramp_start+2
+	orl	%ebx, jmp_32-mptramp_start+2
+	orl	%ebx, jmp_64-mptramp_start+1
 
-	.data
-	ALIGN_DATA				/* just to be sure */
+	/*
+	 * Load the descriptor table pointer.  We'll need it when running
+	 * in 16 bit protected mode.
+	 */
+	lgdt	lgdt_desc-mptramp_start
 
-BOOTMP1:
+	/* Enable protected mode */
+	movl	$CR0_PE, %eax
+	mov	%eax, %cr0 
 
-NON_GPROF_ENTRY(bootMP)
-	.code16		
-	cli
-	CHECKPOINT(0x34, 1)
-	/* First guarantee a 'clean slate' */
-	xorl	%eax, %eax
-	movl	%eax, %ebx
-	movl	%eax, %ecx
- 	movl	%eax, %edx
-	movl	%eax, %esi
-	movl	%eax, %edi
+	/*
+	 * Now execute a far jump to turn on protected mode.  This
+	 * causes the segment registers to turn into selectors and causes
+	 * %cs to be loaded from the gdt.
+	 *
+	 * The following instruction is:
+	 * ljmpl $bootcode-gdt, $protmode-mptramp_start
+	 * but gas cannot assemble that.  And besides, we patch the targets
+	 * in early startup and its a little clearer what we are patching.
+	 */
+jmp_32:
+	.byte	0x66			/* size override to 32 bits */
+	.byte	0xea			/* opcode for far jump */
+	.long	protmode-mptramp_start	/* offset in segment */
+	.word	bootcode-gdt		/* index in gdt for 32 bit code */
 
-	/* set up data segments */
-	mov	%cs, %ax
+	/*
+	 * At this point, we are running in 32 bit legacy protected mode.
+	 */
+	.code32
+protmode:
+	mov	$bootdata-gdt, %eax
 	mov	%ax, %ds
-	mov	%ax, %es
-	mov	%ax, %fs
-	mov	%ax, %gs
-	mov	%ax, %ss
-	mov	$(boot_stk-bootMP), %esp
 
-	/* Now load the global descriptor table */
-	lgdt	MP_GDTptr-bootMP
+	/* Turn on the PAE, PSE and PGE bits for when paging is enabled */
+	mov	%cr4, %eax
+	orl	$(CR4_PAE | CR4_PSE), %eax
+	mov	%eax, %cr4
 
-	/* Enable protected mode */
-	movl	%cr0, %eax
-	orl	$CR0_PE, %eax
-	movl	%eax, %cr0 
+	/*
+	 * Enable EFER.LME so that we get long mode when all the prereqs are
+	 * in place.  In this case, it turns on when CR0_PG is finally enabled.
+	 * Pick up a few other EFER bits that we'll use need we're here.
+	 */
+	movl	$MSR_EFER, %ecx
+	rdmsr
+	orl	$EFER_LME | EFER_SCE | EFER_NXE, %eax
+	wrmsr
 
 	/*
-	 * make intrasegment jump to flush the processor pipeline and
-	 * reload CS register
+	 * Point to the embedded page tables for startup.  Note that this
+	 * only gets accessed after we're actually in 64 bit mode, however
+	 * we can only set the bottom 32 bits of %cr3 in this state.  This
+	 * means we are required to use a temporary page table that is below
+	 * the 4GB limit.  %ebx is still our relocation base.  We could just
+	 * subtract 3 * PAGE_SIZE, but that would be too easy.
 	 */
-	pushl	$0x18
-	pushl	$(protmode-bootMP)
-	lretl
+	leal	mptramp_pagetables-mptramp_start(%ebx),%eax
+	movl	(%eax), %eax
+	mov	%eax, %cr3
 
-       .code32		
-protmode:
-	CHECKPOINT(0x35, 2)
+	/*
+	 * Finally, switch to long bit mode by enabling paging.  We have
+	 * to be very careful here because all the segmentation disappears
+	 * out from underneath us.  The spec says we can depend on the
+	 * subsequent pipelined branch to execute, but *only if* everthing
+	 * is still identity mapped.  If any mappings change, the pipeline
+	 * will flush.
+	 */
+	mov	%cr0, %eax
+	orl	$CR0_PG, %eax
+	mov	%eax, %cr0
 
 	/*
-	 * we are NOW running for the first time with %eip
-	 * having the full physical address, BUT we still
-	 * are using a segment descriptor with the origin
-	 * not matching the booting kernel.
+	 * At this point paging is enabled, and we are in "compatability" mode.
+	 * We do another far jump to reload %cs with the 64 bit selector.
+	 * %cr3 points to a 4-level page table page.
+	 * We cannot yet jump all the way to the kernel because we can only
+	 * specify a 32 bit linear address.  So, yet another trampoline.
 	 *
- 	 * SO NOW... for the BIG Jump into kernel's segment
-	 * and physical text above 1 Meg.
+	 * The following instruction is:
+	 * ljmp $kernelcode-gdt, $tramp_64-mptramp_start
+	 * but gas cannot assemble that.  And besides, we patch the targets
+	 * in early startup and its a little clearer what we are patching.
 	 */
-	mov	$0x10, %ebx
-	movw	%bx, %ds
-	movw	%bx, %es
-	movw	%bx, %fs
-	movw	%bx, %gs
-	movw	%bx, %ss
-
-	.globl	bigJump
-bigJump:
-	/* this will be modified by mpInstallTramp() */
-	ljmp	$0x08, $0			/* far jmp to MPentry() */
-	
-dead:	hlt /* We should never get here */
-	jmp	dead
-
-/*
- * MP boot strap Global Descriptor Table
- */
-	.p2align 4
-	.globl	MP_GDT
-	.globl	bootCodeSeg
-	.globl	bootDataSeg
-MP_GDT:
-
-nulldesc:		/* offset = 0x0 */
+jmp_64:
+	.byte	0xea			/* opcode for far jump */
+	.long	tramp_64-mptramp_start	/* offset in segment */
+	.word	kernelcode-gdt		/* index in gdt for 64 bit code */
 
-	.word	0x0	
-	.word	0x0	
-	.byte	0x0	
-	.byte	0x0	
-	.byte	0x0	
-	.byte	0x0	
-
-kernelcode:		/* offset = 0x08 */
-
-	.word	0xffff	/* segment limit 0..15 */
-	.word	0x0000	/* segment base 0..15 */
-	.byte	0x0	/* segment base 16..23; set for 0K */
-	.byte	0x9f	/* flags; Type	*/
-	.byte	0xcf	/* flags; Limit	*/
-	.byte	0x0	/* segment base 24..32 */
-
-kerneldata:		/* offset = 0x10 */
-
-	.word	0xffff	/* segment limit 0..15 */
-	.word	0x0000	/* segment base 0..15 */
-	.byte	0x0	/* segment base 16..23; set for 0k */
-	.byte	0x93	/* flags; Type  */
-	.byte	0xcf	/* flags; Limit */
-	.byte	0x0	/* segment base 24..32 */
+	/*
+	 * Yeehar!  We're running in 64 bit mode!  We can mostly ignore our
+	 * segment registers, and get on with it.
+	 * Note that we are running at the correct virtual address, but with
+	 * a 1:1 1GB mirrored mapping over entire address space.  We had better
+	 * switch to a real %cr3 promptly so that we can get to the direct map
+	 * space. Remember that jmp is relative and that we've been relocated,
+	 * so use an indirect jump.
+	 */
+	.code64
+tramp_64:
+	movabsq	$entry_64,%rax		/* 64 bit immediate load */
+	jmp	*%rax
 
-bootcode:		/* offset = 0x18 */
+	.p2align 4,0
+gdt:
+	/*
+	 * All segment descriptor tables start with a null descriptor
+	 */
+	.long	0x00000000
+	.long	0x00000000
 
-	.word	0xffff	/* segment limit 0..15 */
-bootCodeSeg:		/* this will be modified by mpInstallTramp() */
-	.word	0x0000	/* segment base 0..15 */
-	.byte	0x00	/* segment base 16...23; set for 0x000xx000 */
-	.byte	0x9e	/* flags; Type  */
-	.byte	0xcf	/* flags; Limit */
-	.byte	0x0	/*segment base 24..32 */
+	/*
+	 * This is the 64 bit long mode code descriptor.  There is no
+	 * 64 bit data descriptor.
+	 */
+kernelcode:
+	.long	0x00000000
+	.long	0x00209800
 
-bootdata:		/* offset = 0x20 */
+	/*
+	 * This is the descriptor for the 32 bit boot code.
+	 * %cs:  +A, +R, -C, DPL=0, +P, +D, +G
+	 * Accessed, Readable, Present, 32 bit, 4G granularity
+	 */
+bootcode:
+	.long	0x0000ffff
+	.long	0x00cf9b00
 
-	.word	0xffff	
-bootDataSeg:		/* this will be modified by mpInstallTramp() */
-	.word	0x0000	/* segment base 0..15 */
-	.byte	0x00	/* segment base 16...23; set for 0x000xx000 */
-	.byte	0x92	
-	.byte	0xcf	
-	.byte	0x0		
+	/*
+	 * This is the descriptor for the 32 bit boot data.
+	 * We load it into %ds and %ss.  The bits for each selector
+	 * are interpreted slightly differently.
+	 * %ds:  +A, +W, -E, DPL=0, +P, +D, +G
+	 * %ss:  +A, +W, -E, DPL=0, +P, +B, +G
+	 * Accessed, Writeable, Expand up, Present, 32 bit, 4GB 
+	 * For %ds, +D means 'default operand size is 32 bit'.
+	 * For %ss, +B means the stack register is %esp rather than %sp.
+	 */
+bootdata:
+	.long	0x0000ffff
+	.long	0x00cf9300
 
-/*
- * GDT pointer for the lgdt call
- */
-	.globl	mp_gdtbase
+gdtend:
 
-MP_GDTptr:	
-mp_gdtlimit:
-	.word	0x0028		
-mp_gdtbase:		/* this will be modified by mpInstallTramp() */
+	/*
+	 * The address of our page table pages that the boot code
+	 * uses to trampoline up to kernel address space.
+	 */
+	.globl	mptramp_pagetables
+mptramp_pagetables:
 	.long	0
 
-	.space	0x100	/* space for boot_stk - 1st temporary stack */
-boot_stk:
+	/*
+	 * The pseudo descriptor for lgdt to use.
+	 */
+lgdt_desc:	
+	.word	gdtend-gdt		/* Length */
+	.long	gdt-mptramp_start	/* Offset plus %ds << 4 */
+
+	.globl	mptramp_end
+mptramp_end:
 
-BOOTMP2:
-	.globl	bootMP_size
-bootMP_size:
-	.long	BOOTMP2 - BOOTMP1
+	/*
+	 * From here on down is executed in the kernel .text section.
+	 *
+	 * Load a real %cr3 that has all the direct map stuff and switches
+	 * off the 1GB replicated mirror.  Load a stack pointer and jump
+	 * into AP startup code in C.
+	 */
+	.text
+	.code64
+	.p2align 4,0
+entry_64:
+	movq	KPML4phys, %rax
+	movq	%rax, %cr3
+	movq	bootSTK, %rsp
+	jmp	init_secondary
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 908e65a..f0a9883 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -27,7 +27,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_mptable_force_htt.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
@@ -57,13 +56,8 @@ __FBSDID("$FreeBSD$");
 
 #define	NAPICID			32	/* Max number of I/O APIC's */
 
-#ifdef PC98
-#define BIOS_BASE		(0xe8000)
-#define BIOS_SIZE		(0x18000)
-#else
 #define BIOS_BASE		(0xf0000)
 #define BIOS_SIZE		(0x10000)
-#endif
 #define BIOS_COUNT		(BIOS_SIZE/4)
 
 typedef	void mptable_entry_handler(u_char *entry, void *arg);
@@ -226,11 +220,12 @@ static int
 mptable_probe(void)
 {
 	int     x;
-	u_long  segment;
+	u_int32_t segment;
 	u_int32_t target;
 
 	/* see if EBDA exists */
-	if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) {
+	segment = (u_int32_t) *(u_short *)(KERNBASE + 0x40e);
+	if (segment != 0) {
 		/* search first 1K of EBDA */
 		target = (u_int32_t) (segment << 4);
 		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
diff --git a/sys/amd64/amd64/nexus.c b/sys/amd64/amd64/nexus.c
index 952ceaf..1fab16b 100644
--- a/sys/amd64/amd64/nexus.c
+++ b/sys/amd64/amd64/nexus.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <machine/bus.h>
+#include <machine/intr_machdep.h>
 #include <sys/rman.h>
 #include <sys/interrupt.h>
 
@@ -64,8 +65,6 @@ __FBSDID("$FreeBSD$");
 #include <isa/isavar.h>
 #include <amd64/isa/isa.h>
 #endif
-#include <amd64/isa/icu.h>
-#include <amd64/isa/intr_machdep.h>
 #include <sys/rtprio.h>
 
 static MALLOC_DEFINE(M_NEXUSDEV, "nexusdev", "Nexus device");
@@ -156,14 +155,11 @@ nexus_probe(device_t dev)
 	 * multi-ISA-bus systems.  PCI interrupts are routed to the ISA
 	 * component, so in a way, PCI can be a partial child of an ISA bus(!).
 	 * APIC interrupts are global though.
-	 *
-	 * XXX We depend on the AT PIC driver correctly claiming IRQ 2
-	 *     to prevent its reuse elsewhere.
 	 */
 	irq_rman.rm_start = 0;
 	irq_rman.rm_type = RMAN_ARRAY;
 	irq_rman.rm_descr = "Interrupt request lines";
-	irq_rman.rm_end = 15;
+	irq_rman.rm_end = NUM_IO_INTS - 1;
 	if (rman_init(&irq_rman)
 	    || rman_manage_region(&irq_rman,
 				  irq_rman.rm_start, irq_rman.rm_end))
@@ -428,7 +424,7 @@ nexus_setup_intr(device_t bus, device_t child, struct resource *irq,
 	if (error)
 		return (error);
 
-	error = inthand_add(device_get_nameunit(child), irq->r_start,
+	error = intr_add_handler(device_get_nameunit(child), irq->r_start,
 	    ihand, arg, flags, cookiep);
 
 	return (error);
@@ -437,7 +433,7 @@ nexus_setup_intr(device_t bus, device_t child, struct resource *irq,
 static int
 nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih)
 {
-	return (inthand_remove(ih));
+	return (intr_remove_handler(ih));
 }
 
 static int
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 4e4c124..b2f0c18 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -118,6 +118,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/user.h>
 #include <sys/vmmeter.h>
 #include <sys/sysctl.h>
+#ifdef SMP
+#include <sys/smp.h>
+#endif
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
@@ -134,6 +137,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 
 #define PMAP_KEEP_PDIRS
 #ifndef PMAP_SHPGPERPROC
@@ -163,6 +169,11 @@ struct pmap kernel_pmap_store;
 LIST_HEAD(pmaplist, pmap);
 static struct pmaplist allpmaps;
 static struct mtx allpmaps_lock;
+#ifdef LAZY_SWITCH
+#ifdef SMP
+static struct mtx lazypmap_lock;
+#endif
+#endif
 
 vm_paddr_t avail_start;		/* PA of first available physical page */
 vm_paddr_t avail_end;		/* PA of last available physical page */
@@ -477,6 +488,11 @@ pmap_bootstrap(firstaddr)
 	kernel_pmap->pm_active = -1;	/* don't allow deactivation */
 	TAILQ_INIT(&kernel_pmap->pm_pvlist);
 	LIST_INIT(&allpmaps);
+#ifdef LAZY_SWITCH
+#ifdef SMP
+	mtx_init(&lazypmap_lock, "lazypmap", NULL, MTX_SPIN);
+#endif
+#endif
 	mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN);
 	mtx_lock_spin(&allpmaps_lock);
 	LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list);
@@ -630,8 +646,121 @@ pmap_track_modified(vm_offset_t va)
 		return 0;
 }
 
+#ifdef SMP
 /*
- * Normal invalidation functions.
+ * For SMP, these functions have to use the IPI mechanism for coherence.
+ */
+void
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
+{
+	u_int cpumask;
+	u_int other_cpus;
+
+	if (smp_started) {
+		if (!(read_rflags() & PSL_I))
+			panic("%s: interrupts disabled", __func__);
+		mtx_lock_spin(&smp_tlb_mtx);
+	} else
+		critical_enter();
+	/*
+	 * We need to disable interrupt preemption but MUST NOT have
+	 * interrupts disabled here.
+	 * XXX we may need to hold schedlock to get a coherent pm_active
+	 * XXX critical sections disable interrupts again
+	 */
+	if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
+		invlpg(va);
+		smp_invlpg(va);
+	} else {
+		cpumask = PCPU_GET(cpumask);
+		other_cpus = PCPU_GET(other_cpus);
+		if (pmap->pm_active & cpumask)
+			invlpg(va);
+		if (pmap->pm_active & other_cpus)
+			smp_masked_invlpg(pmap->pm_active & other_cpus, va);
+	}
+	if (smp_started)
+		mtx_unlock_spin(&smp_tlb_mtx);
+	else
+		critical_exit();
+}
+
+void
+pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+{
+	u_int cpumask;
+	u_int other_cpus;
+	vm_offset_t addr;
+
+	if (smp_started) {
+		if (!(read_rflags() & PSL_I))
+			panic("%s: interrupts disabled", __func__);
+		mtx_lock_spin(&smp_tlb_mtx);
+	} else
+		critical_enter();
+	/*
+	 * We need to disable interrupt preemption but MUST NOT have
+	 * interrupts disabled here.
+	 * XXX we may need to hold schedlock to get a coherent pm_active
+	 * XXX critical sections disable interrupts again
+	 */
+	if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
+		for (addr = sva; addr < eva; addr += PAGE_SIZE)
+			invlpg(addr);
+		smp_invlpg_range(sva, eva);
+	} else {
+		cpumask = PCPU_GET(cpumask);
+		other_cpus = PCPU_GET(other_cpus);
+		if (pmap->pm_active & cpumask)
+			for (addr = sva; addr < eva; addr += PAGE_SIZE)
+				invlpg(addr);
+		if (pmap->pm_active & other_cpus)
+			smp_masked_invlpg_range(pmap->pm_active & other_cpus,
+			    sva, eva);
+	}
+	if (smp_started)
+		mtx_unlock_spin(&smp_tlb_mtx);
+	else
+		critical_exit();
+}
+
+void
+pmap_invalidate_all(pmap_t pmap)
+{
+	u_int cpumask;
+	u_int other_cpus;
+
+	if (smp_started) {
+		if (!(read_rflags() & PSL_I))
+			panic("%s: interrupts disabled", __func__);
+		mtx_lock_spin(&smp_tlb_mtx);
+	} else
+		critical_enter();
+	/*
+	 * We need to disable interrupt preemption but MUST NOT have
+	 * interrupts disabled here.
+	 * XXX we may need to hold schedlock to get a coherent pm_active
+	 * XXX critical sections disable interrupts again
+	 */
+	if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
+		invltlb();
+		smp_invltlb();
+	} else {
+		cpumask = PCPU_GET(cpumask);
+		other_cpus = PCPU_GET(other_cpus);
+		if (pmap->pm_active & cpumask)
+			invltlb();
+		if (pmap->pm_active & other_cpus)
+			smp_masked_invltlb(pmap->pm_active & other_cpus);
+	}
+	if (smp_started)
+		mtx_unlock_spin(&smp_tlb_mtx);
+	else
+		critical_exit();
+}
+#else /* !SMP */
+/*
+ * Normal, non-SMP, invalidation functions.
  * We inline these within pmap.c for speed.
  */
 PMAP_INLINE void
@@ -659,6 +788,7 @@ pmap_invalidate_all(pmap_t pmap)
 	if (pmap == kernel_pmap || pmap->pm_active)
 		invltlb();
 }
+#endif /* !SMP */
 
 /*
  * Are we current address space or kernel?
@@ -1208,6 +1338,93 @@ retry:
  * Pmap allocation/deallocation routines.
  ***************************************************/
 
+#ifdef LAZY_SWITCH
+#ifdef SMP
+/*
+ * Deal with a SMP shootdown of other users of the pmap that we are
+ * trying to dispose of.  This can be a bit hairy.
+ */
+static u_int *lazymask;
+static register_t lazyptd;
+static volatile u_int lazywait;
+
+void pmap_lazyfix_action(void);
+
+void
+pmap_lazyfix_action(void)
+{
+	u_int mymask = PCPU_GET(cpumask);
+
+	if (rcr3() == lazyptd)
+		load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+	atomic_clear_int(lazymask, mymask);
+	atomic_store_rel_int(&lazywait, 1);
+}
+
+static void
+pmap_lazyfix_self(u_int mymask)
+{
+
+	if (rcr3() == lazyptd)
+		load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+	atomic_clear_int(lazymask, mymask);
+}
+
+
+static void
+pmap_lazyfix(pmap_t pmap)
+{
+	u_int mymask = PCPU_GET(cpumask);
+	u_int mask;
+	register u_int spins;
+
+	while ((mask = pmap->pm_active) != 0) {
+		spins = 50000000;
+		mask = mask & -mask;	/* Find least significant set bit */
+		mtx_lock_spin(&lazypmap_lock);
+		lazyptd = vtophys(pmap->pm_pml4);
+		if (mask == mymask) {
+			lazymask = &pmap->pm_active;
+			pmap_lazyfix_self(mymask);
+		} else {
+			atomic_store_rel_long((u_long *)&lazymask,
+			    (u_long)&pmap->pm_active);
+			atomic_store_rel_int(&lazywait, 0);
+			ipi_selected(mask, IPI_LAZYPMAP);
+			while (lazywait == 0) {
+				ia32_pause();
+				if (--spins == 0)
+					break;
+			}
+		}
+		mtx_unlock_spin(&lazypmap_lock);
+		if (spins == 0)
+			printf("pmap_lazyfix: spun for 50000000\n");
+	}
+}
+
+#else	/* SMP */
+
+/*
+ * Cleaning up on uniprocessor is easy.  For various reasons, we're
+ * unlikely to have to even execute this code, including the fact
+ * that the cleanup is deferred until the parent does a wait(2), which
+ * means that another userland process has run.
+ */
+static void
+pmap_lazyfix(pmap_t pmap)
+{
+	u_long cr3;
+
+	cr3 = vtophys(pmap->pm_pml4);
+	if (cr3 == rcr3()) {
+		load_cr3(PCPU_GET(curpcb)->pcb_cr3);
+		pmap->pm_active &= ~(PCPU_GET(cpumask));
+	}
+}
+#endif	/* SMP */
+#endif
+
 /*
  * Release any resources held by the given physical map.
  * Called when a pmap initialized by pmap_pinit is being released.
@@ -1222,6 +1439,9 @@ pmap_release(pmap_t pmap)
 	    ("pmap_release: pmap resident count %ld != 0",
 	    pmap->pm_stats.resident_count));
 
+#ifdef LAZY_SWITCH
+	pmap_lazyfix(pmap);
+#endif
 	mtx_lock_spin(&allpmaps_lock);
 	LIST_REMOVE(pmap, pm_list);
 	mtx_unlock_spin(&allpmaps_lock);
@@ -2777,12 +2997,21 @@ void
 pmap_activate(struct thread *td)
 {
 	struct proc *p = td->td_proc;
-	pmap_t	pmap;
+	pmap_t	pmap, oldpmap;
 	u_int64_t  cr3;
 
 	critical_enter();
 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
+	oldpmap = PCPU_GET(curpmap);
+#ifdef SMP
+if (oldpmap)	/* XXX FIXME */
+	atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
+	atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
+#else
+if (oldpmap)	/* XXX FIXME */
+	oldpmap->pm_active &= ~PCPU_GET(cpumask);
 	pmap->pm_active |= PCPU_GET(cpumask);
+#endif
 	cr3 = vtophys(pmap->pm_pml4);
 	/* XXXKSE this is wrong.
 	 * pmap_activate is for the current thread on the current cpu
diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S
index 94d7bba..2c0ddf1 100644
--- a/sys/amd64/amd64/support.S
+++ b/sys/amd64/amd64/support.S
@@ -37,10 +37,21 @@
 #include "opt_ddb.h"
 
 #include <machine/asmacros.h>
+#include <machine/intr_machdep.h>
 #include <machine/pmap.h>
 
 #include "assym.s"
 
+	ALIGN_DATA
+	.globl	intrcnt, eintrcnt
+intrcnt:
+	.space	INTRCNT_COUNT * 4
+eintrcnt:
+
+	.globl	intrnames, eintrnames
+intrnames:
+	.space	INTRCNT_COUNT * (MAXCOMLEN + 1)
+eintrnames:
 
 	.text
 
@@ -302,6 +313,9 @@ ENTRY(casuptr)
 	ja	fusufault
 
 	movq	%rsi, %rax			/* old */
+#ifdef SMP
+	lock
+#endif
 	cmpxchgq %rdx, (%rdi)			/* new = %rdx */
 
 	/*
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index 65ae8f9..0e6b95b 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -80,13 +80,14 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 
 #include <machine/cpu.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 #include <machine/tss.h>
 
-#include <amd64/isa/icu.h>
-#include <amd64/isa/intr_machdep.h>
-
 #include <ddb/ddb.h>
 
 extern void trap(struct trapframe frame);
@@ -564,6 +565,11 @@ trap_fatal(frame, eva)
 		printf("\n\nFatal trap %d: %s while in %s mode\n",
 			type, trap_msg[type],
 			ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
+#ifdef SMP
+	/* two separate prints in case of a trap on an unmapped page */
+	printf("cpuid = %d; ", PCPU_GET(cpuid));
+	printf("apic id = %02x\n", PCPU_GET(apic_id));
+#endif
 	if (type == T_PAGEFLT) {
 		printf("fault virtual address	= 0x%lx\n", eva);
 		printf("fault code		= %s %s, %s\n",
@@ -631,6 +637,11 @@ void
 dblfault_handler()
 {
 	printf("\nFatal double fault\n");
+#ifdef SMP
+	/* two separate prints in case of a trap on an unmapped page */
+	printf("cpuid = %d; ", PCPU_GET(cpuid));
+	printf("apic id = %02x\n", PCPU_GET(apic_id));
+#endif
 	panic("double fault");
 }
 
diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c
index 5485511..6a5b17c 100644
--- a/sys/amd64/amd64/tsc.c
+++ b/sys/amd64/amd64/tsc.c
@@ -77,14 +77,26 @@ init_TSC(void)
 	tsc_freq = tscval[1] - tscval[0];
 	if (bootverbose)
 		printf("TSC clock: %lu Hz\n", tsc_freq);
-
-	return;
 }
 
+
 void
 init_TSC_tc(void)
 {
 
+#ifdef SMP
+	/*
+	 * We can not use the TSC in SMP mode unless the TSCs on all CPUs
+	 * are somehow synchronized.  Some hardware configurations do
+	 * this, but we have no way of determining whether this is the
+	 * case, so we do not use the TSC in multi-processor systems
+	 * unless the user indicated (by setting kern.timecounter.smp_tsc
+	 * to 1) that he believes that his TSCs are synchronized.
+	 */
+	if (mp_ncpus > 1 && !smp_tsc)
+		tsc_timecounter.tc_quality = -100;
+#endif
+
 	if (tsc_freq != 0 && !tsc_is_broken) {
 		tsc_timecounter.tc_frequency = tsc_freq;
 		tc_init(&tsc_timecounter);
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index 9b6bc1f..c4f583f 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/sf_buf.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/unistd.h>
 
@@ -80,6 +81,11 @@ __FBSDID("$FreeBSD$");
 #include <amd64/isa/isa.h>
 
 static void	cpu_reset_real(void);
+#ifdef SMP
+static void	cpu_reset_proxy(void);
+static u_int	cpu_reset_proxyid;
+static volatile u_int	cpu_reset_proxy_active;
+#endif
 static void	sf_buf_init(void *arg);
 SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL)
 
@@ -336,10 +342,69 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
  * Force reset the processor by invalidating the entire address space!
  */
 
+#ifdef SMP
+static void
+cpu_reset_proxy()
+{
+
+	cpu_reset_proxy_active = 1;
+	while (cpu_reset_proxy_active == 1)
+		;	 /* Wait for other cpu to see that we've started */
+	stop_cpus((1<<cpu_reset_proxyid));
+	printf("cpu_reset_proxy: Stopped CPU %d\n", cpu_reset_proxyid);
+	DELAY(1000000);
+	cpu_reset_real();
+}
+#endif
+
 void
 cpu_reset()
 {
+#ifdef SMP
+	if (smp_active == 0) {
+		cpu_reset_real();
+		/* NOTREACHED */
+	} else {
+
+		u_int map;
+		int cnt;
+		printf("cpu_reset called on cpu#%d\n", PCPU_GET(cpuid));
+
+		map = PCPU_GET(other_cpus) & ~ stopped_cpus;
+
+		if (map != 0) {
+			printf("cpu_reset: Stopping other CPUs\n");
+			stop_cpus(map);		/* Stop all other CPUs */
+		}
+
+		if (PCPU_GET(cpuid) == 0) {
+			DELAY(1000000);
+			cpu_reset_real();
+			/* NOTREACHED */
+		} else {
+			/* We are not BSP (CPU #0) */
+
+			cpu_reset_proxyid = PCPU_GET(cpuid);
+			cpustop_restartfunc = cpu_reset_proxy;
+			cpu_reset_proxy_active = 0;
+			printf("cpu_reset: Restarting BSP\n");
+			started_cpus = (1<<0);		/* Restart CPU #0 */
+
+			cnt = 0;
+			while (cpu_reset_proxy_active == 0 && cnt < 10000000)
+				cnt++;	/* Wait for BSP to announce restart */
+			if (cpu_reset_proxy_active == 0)
+				printf("cpu_reset: Failed to restart BSP\n");
+			enable_intr();
+			cpu_reset_proxy_active = 2;
+
+			while (1);
+			/* NOTREACHED */
+		}
+	}
+#else
 	cpu_reset_real();
+#endif
 }
 
 static void
diff --git a/sys/amd64/ia32/ia32_syscall.c b/sys/amd64/ia32/ia32_syscall.c
index 991c85b..e32cfcd 100644
--- a/sys/amd64/ia32/ia32_syscall.c
+++ b/sys/amd64/ia32/ia32_syscall.c
@@ -76,11 +76,9 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 
 #include <machine/cpu.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 
-#include <amd64/isa/icu.h>
-#include <amd64/isa/intr_machdep.h>
-
 #define	IDTVEC(name)	__CONCAT(X,name)
 
 extern inthand_t IDTVEC(int0x80_syscall), IDTVEC(rsvd);
diff --git a/sys/amd64/include/apicvar.h b/sys/amd64/include/apicvar.h
index 9d5cdff..4dea040 100644
--- a/sys/amd64/include/apicvar.h
+++ b/sys/amd64/include/apicvar.h
@@ -32,6 +32,8 @@
 #ifndef _MACHINE_APICVAR_H_
 #define _MACHINE_APICVAR_H_
 
+#include <machine/segments.h>
+
 /*
  * Local && I/O APIC variable definitions.
  */
@@ -156,7 +158,7 @@ int	lapic_intr_pending(u_int vector);
 void	lapic_ipi_raw(register_t icrlo, u_int dest);
 void	lapic_ipi_vectored(u_int vector, int dest);
 int	lapic_ipi_wait(int delay);
-void	lapic_handle_intr(struct intrframe frame);
+void	lapic_handle_intr(void *cookie, struct intrframe frame);
 void	lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id);
 int	lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked);
 int	lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode);
diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h
index 9893e5f..404287e 100644
--- a/sys/amd64/include/asmacros.h
+++ b/sys/amd64/include/asmacros.h
@@ -138,4 +138,13 @@
 #define MEXITCOUNT
 #endif /* GPROF */
 
+#ifdef LOCORE
+/*
+ * Convenience macros for declaring interrupt entry points.
+ */
+#define	IDTVEC(name)	ALIGN_TEXT; .globl __CONCAT(X,name); \
+			.type __CONCAT(X,name),@function; __CONCAT(X,name):
+
+#endif /* LOCORE */
+
 #endif /* !_MACHINE_ASMACROS_H_ */
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index 0c4d9b0..bc10631 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -82,7 +82,7 @@ void		atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
  * For userland, assume the SMP case and use lock prefixes so that
  * the binaries will run on both types of systems.
  */
-#if !defined(_KERNEL)
+#if defined(SMP) || !defined(_KERNEL)
 #define MPLOCKED	lock ;
 #else
 #define MPLOCKED
diff --git a/sys/amd64/include/clock.h b/sys/amd64/include/clock.h
index 359e173..9e3e4e5 100644
--- a/sys/amd64/include/clock.h
+++ b/sys/amd64/include/clock.h
@@ -15,6 +15,7 @@
  * XXX large parts of the driver and its interface are misplaced.
  */
 extern int	adjkerntz;
+extern int	clkintr_pending;
 extern int	disable_rtc_set;
 extern int	pscnt;
 extern int	psdiv;
diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h
index 38b07eb..2f32cfd 100644
--- a/sys/amd64/include/md_var.h
+++ b/sys/amd64/include/md_var.h
@@ -38,6 +38,7 @@
 
 extern	long	Maxmem;
 extern	u_long	atdevbase;	/* offset in virtual memory of ISA io mem */
+extern	u_int	basemem;
 extern	int	busdma_swi_pending;
 extern	u_int	cpu_exthigh;
 extern	u_int	cpu_feature;
@@ -50,7 +51,8 @@ extern	uint16_t *elan_mmcr;
 extern	char	kstack[];
 extern	char	sigcode[];
 extern	int	szsigcode;
-extern	u_int	basemem;
+
+extern	struct pcpu __pcpu[];
 
 typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss);
 struct	thread;
@@ -66,6 +68,7 @@ void	enable_sse(void);
 void	fillw(int /*u_short*/ pat, void *base, size_t cnt);
 void	pagezero(void *addr);
 int	is_physical_memory(vm_offset_t addr);
+int	isa_nmi(int cd);
 void	setidt(int idx, alias_for_inthand_t *func, int typ, int dpl, int ist);
 void	fpstate_drop(struct thread *td);
 
diff --git a/sys/amd64/include/param.h b/sys/amd64/include/param.h
index 293f909..7c3a831 100644
--- a/sys/amd64/include/param.h
+++ b/sys/amd64/include/param.h
@@ -83,7 +83,11 @@
 #define	MACHINE_ARCH	"amd64"
 #endif
 
+#ifdef SMP
+#define MAXCPU		8
+#else
 #define MAXCPU		1
+#endif
 
 #define	ALIGNBYTES		_ALIGNBYTES
 #define	ALIGN(p)		_ALIGN(p)
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index 7ebfbba..9e0383a 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -40,7 +40,11 @@
  */
 #define	PCPU_MD_FIELDS							\
 	struct	pcpu *pc_prvspace;	/* Self-reference */		\
+	struct	pmap *pc_curpmap;					\
+	struct	amd64tss *pc_tssp;					\
+	register_t pc_rsp0;						\
 	register_t pc_scratch_rsp;	/* User %rsp in syscall */	\
+	u_int	pc_apic_id;						\
 	u_int   pc_acpi_id		/* ACPI CPU id */
 
 #if defined(lint)
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index 77a25a4..d6ba652 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -221,7 +221,8 @@ struct md_page {
 struct pmap {
 	pml4_entry_t		*pm_pml4;	/* KVA of level 4 page table */
 	TAILQ_HEAD(,pv_entry)	pm_pvlist;	/* list of mappings in pmap */
-	u_long			pm_active;	/* active on cpus */
+	u_int			pm_active;	/* active on cpus */
+	/* spare u_int here due to padding */
 	struct pmap_statistics	pm_stats;	/* pmap statistics */
 	LIST_ENTRY(pmap) 	pm_list;	/* List of all pmaps */
 };
diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h
index d924eb3..806e9d5 100644
--- a/sys/amd64/include/segments.h
+++ b/sys/amd64/include/segments.h
@@ -49,6 +49,7 @@
  * Selectors
  */
 
+#define	SEL_RPL_MASK	3	/* requester priv level */
 #define	ISPL(s)	((s)&3)		/* what is the priority level of a selector */
 #define	SEL_KPL	0		/* kernel priority level */
 #define	SEL_UPL	3		/* user priority level */
@@ -191,6 +192,7 @@ struct region_descriptor {
 #define	IDT_AC		17	/* #AC: Alignment Check */
 #define	IDT_MC		18	/* #MC: Machine Check */
 #define	IDT_XF		19	/* #XF: SIMD Floating-Point Exception */
+#define	IDT_IO_INTS	NRSVIDT	/* Base of IDT entries for I/O interrupts. */
 #define	IDT_SYSCALL	0x80	/* System Call Interrupt Vector */
 
 /*
@@ -210,6 +212,7 @@ struct region_descriptor {
 extern struct user_segment_descriptor gdt[];
 extern struct soft_segment_descriptor gdt_segs[];
 extern struct gate_descriptor *idt;
+extern struct region_descriptor r_gdt, r_idt;
 
 void	lgdt(struct region_descriptor *rdp);
 void	sdtossd(struct user_segment_descriptor *sdp,
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 3d8d117..c6e7fcf 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -13,4 +13,63 @@
 #ifndef _MACHINE_SMP_H_
 #define _MACHINE_SMP_H_
 
+#ifdef _KERNEL
+
+#ifdef SMP
+
+#ifndef LOCORE
+
+#include <sys/bus.h>
+#include <machine/frame.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+
+/* global symbols in mpboot.S */
+extern char			mptramp_start[];
+extern char			mptramp_end[];
+extern u_int32_t		mptramp_pagetables;
+
+/* global data in mp_machdep.c */
+extern int			mp_naps;
+extern int			boot_cpu_id;
+extern struct pcb		stoppcbs[];
+extern struct mtx		smp_tlb_mtx;
+
+/* IPI handlers */
+inthand_t
+	IDTVEC(invltlb),	/* TLB shootdowns - global */
+	IDTVEC(invlpg),		/* TLB shootdowns - 1 page */
+	IDTVEC(invlrng),	/* TLB shootdowns - page range */
+	IDTVEC(hardclock),	/* Forward hardclock() */
+	IDTVEC(statclock),	/* Forward statclock() */
+	IDTVEC(cpuast),		/* Additional software trap on other cpu */ 
+	IDTVEC(cpustop),	/* CPU stops & waits to be restarted */
+	IDTVEC(rendezvous),	/* handle CPU rendezvous */
+	IDTVEC(lazypmap);	/* handle lazy pmap release */
+
+/* functions in mp_machdep.c */
+void	cpu_add(u_int apic_id, char boot_cpu);
+void	init_secondary(void);
+void	ipi_selected(u_int cpus, u_int ipi);
+void	ipi_all(u_int ipi);
+void	ipi_all_but_self(u_int ipi);
+void	ipi_self(u_int ipi);
+void	forward_statclock(void);
+void	forwarded_statclock(struct clockframe frame);
+void	forward_hardclock(void);
+void	forwarded_hardclock(struct clockframe frame);
+u_int	mp_bootaddress(u_int);
+int	mp_grab_cpu_hlt(void);
+void	smp_invlpg(vm_offset_t addr);
+void	smp_masked_invlpg(u_int mask, vm_offset_t addr);
+void	smp_invlpg_range(vm_offset_t startva, vm_offset_t endva);
+void	smp_masked_invlpg_range(u_int mask, vm_offset_t startva,
+	    vm_offset_t endva);
+void	smp_invltlb(void);
+void	smp_masked_invltlb(u_int mask);
+
+#endif /* !LOCORE */
+#endif /* SMP */
+
+#endif /* _KERNEL */
 #endif /* _MACHINE_SMP_H_ */
diff --git a/sys/amd64/include/tss.h b/sys/amd64/include/tss.h
index aa60ba0..24d31b3 100644
--- a/sys/amd64/include/tss.h
+++ b/sys/amd64/include/tss.h
@@ -69,7 +69,7 @@ struct amd64tss {
 };
 
 #ifdef _KERNEL
-extern struct amd64tss common_tss;
+extern struct amd64tss common_tss[];
 #endif
 
 #endif /* _MACHINE_TSS_H_ */
diff --git a/sys/amd64/isa/atpic.c b/sys/amd64/isa/atpic.c
index ba44094..64fa45e3 100644
--- a/sys/amd64/isa/atpic.c
+++ b/sys/amd64/isa/atpic.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/proc.h>
 
 #include <machine/cpufunc.h>
 #include <machine/frame.h>
@@ -52,26 +53,14 @@ __FBSDID("$FreeBSD$");
 #include <machine/resource.h>
 #include <machine/segments.h>
 
-#include <i386/isa/icu.h>
-#ifdef PC98
-#include <pc98/pc98/pc98.h>
-#else
-#include <i386/isa/isa.h>
-#endif
+#include <amd64/isa/icu.h>
+#include <amd64/isa/isa.h>
+
 #include <isa/isavar.h>
 
 #define	MASTER	0
 #define	SLAVE	1
 
-/* XXX: Magic numbers */
-#ifdef PC98
-#ifdef AUTO_EOI_1
-#define	MASTER_MODE	0x1f	/* Master auto EOI, 8086 mode */
-#else
-#define	MASTER_MODE	0x1d	/* Master 8086 mode */
-#endif
-#define	SLAVE_MODE	9	/* 8086 mode */
-#else /* IBM-PC */
 #ifdef AUTO_EOI_1
 #define	MASTER_MODE	(ICW4_8086 | ICW4_AEOI)
 #else
@@ -82,7 +71,6 @@ __FBSDID("$FreeBSD$");
 #else
 #define	SLAVE_MODE	ICW4_8086
 #endif
-#endif /* PC98 */
 
 static void	atpic_init(void *dummy);
 
@@ -252,13 +240,7 @@ i8259_init(struct atpic *pic, int slave)
 
 	/* Reset the PIC and program with next four bytes. */
 	mtx_lock_spin(&icu_lock);
-#ifdef DEV_MCA
-	/* MCA uses level triggered interrupts. */
-	if (MCA_system)
-		outb(pic->at_ioaddr, ICW1_RESET | ICW1_IC4 | ICW1_LTIM);
-	else
-#endif
-		outb(pic->at_ioaddr, ICW1_RESET | ICW1_IC4);
+	outb(pic->at_ioaddr, ICW1_RESET | ICW1_IC4);
 	imr_addr = pic->at_ioaddr + ICU_IMR_OFFSET;
 
 	/* Start vector. */
@@ -286,11 +268,9 @@ i8259_init(struct atpic *pic, int slave)
 	/* Reset is finished, default to IRR on read. */
 	outb(pic->at_ioaddr, OCW3_SEL | OCW3_RR);
 
-#ifndef PC98
 	/* OCW2_L1 sets priority order to 3-7, 0-2 (com2 first). */
 	if (!slave)
 		outb(pic->at_ioaddr, OCW2_R | OCW2_SL | OCW2_L1);
-#endif
 	mtx_unlock_spin(&icu_lock);
 }
 
@@ -317,21 +297,36 @@ atpic_init(void *dummy __unused)
 			continue;
 		ai = &atintrs[i];
 		setidt(((struct atpic *)ai->at_intsrc.is_pic)->at_intbase +
-		    ai->at_irq, ai->at_intr, SDT_SYS386IGT, SEL_KPL,
-		    GSEL(GCODE_SEL, SEL_KPL));
+		    ai->at_irq, ai->at_intr, SDT_SYSIGT, SEL_KPL, 0);
 		intr_register_source(&ai->at_intsrc);
 	}
 }
 SYSINIT(atpic_init, SI_SUB_INTR, SI_ORDER_SECOND + 1, atpic_init, NULL)
 
 void
-atpic_handle_intr(struct intrframe iframe)
+atpic_handle_intr(void *cookie, struct intrframe iframe)
 {
 	struct intsrc *isrc;
-
-	KASSERT((uint)iframe.if_vec < ICU_LEN,
-	    ("unknown int %d\n", iframe.if_vec));
-	isrc = &atintrs[iframe.if_vec].at_intsrc;
+	int vec = (uintptr_t)cookie;
+
+	KASSERT(vec < ICU_LEN, ("unknown int %d\n", vec));
+	isrc = &atintrs[vec].at_intsrc;
+	if (vec == 7 || vec == 15) {
+		int port, isr;
+
+		/*
+		 * Read the ISR register to see if IRQ 7/15 is really
+		 * pending.  Reset read register back to IRR when done.
+		 */
+		port = ((struct atpic *)isrc->is_pic)->at_ioaddr;
+		mtx_lock_spin(&icu_lock);
+		outb(port, OCW3_SEL | OCW3_RR | OCW3_RIS);
+		isr = inb(port);
+		outb(port, OCW3_SEL | OCW3_RR);
+		mtx_unlock_spin(&icu_lock);
+		if ((isr & IRQ7) == 0)
+			return;
+	}
 	intr_execute_handlers(isrc, &iframe);
 }
 
@@ -398,9 +393,7 @@ static driver_t atpic_driver = {
 static devclass_t atpic_devclass;
 
 DRIVER_MODULE(atpic, isa, atpic_driver, atpic_devclass, 0, 0);
-#ifndef PC98
 DRIVER_MODULE(atpic, acpi, atpic_driver, atpic_devclass, 0, 0);
-#endif
 
 /*
  * Return a bitmap of the current interrupt requests.  This is 8259-specific
diff --git a/sys/amd64/isa/atpic_vector.S b/sys/amd64/isa/atpic_vector.S
index e3dc38f..e41071df 100644
--- a/sys/amd64/isa/atpic_vector.S
+++ b/sys/amd64/isa/atpic_vector.S
@@ -41,12 +41,8 @@
  */
 
 #include <machine/asmacros.h>
-#include <i386/isa/icu.h>
-#ifdef PC98
-#include <pc98/pc98/pc98.h>
-#else
-#include <i386/isa/isa.h>
-#endif
+#include <amd64/isa/icu.h>
+#include <amd64/isa/isa.h>
 
 #include "assym.s"
 
@@ -57,23 +53,28 @@
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
 IDTVEC(vec_name) ;							\
-	pushl	$0 ;		/* dummy error code */			\
-	pushl	$0 ;		/* dummy trap type */			\
-	pushal ;		/* 8 ints */				\
-	pushl	%ds ;		/* save data and extra segments ... */	\
-	pushl	%es ;							\
-	pushl	%fs ;							\
-	mov	$KDSEL,%ax ;	/* load kernel ds, es and fs */		\
-	mov	%ax,%ds ;						\
-	mov	%ax,%es ;						\
-	mov	$KPSEL,%ax ;						\
-	mov	%ax,%fs ;						\
-;									\
+	subq	$TF_RIP,%rsp ;	/* skip dummy tf_err and tf_trapno */	\
+	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
+	jz	1f ;		/* Yes, dont swapgs again */		\
+	swapgs ;							\
+1:	movq	%rdi,TF_RDI(%rsp) ;					\
+	movq	%rsi,TF_RSI(%rsp) ;					\
+	movq	%rdx,TF_RDX(%rsp) ;					\
+	movq	%rcx,TF_RCX(%rsp) ;					\
+	movq	%r8,TF_R8(%rsp) ;					\
+	movq	%r9,TF_R9(%rsp) ;					\
+	movq	%rax,TF_RAX(%rsp) ;					\
+	movq	%rbx,TF_RBX(%rsp) ;					\
+	movq	%rbp,TF_RBP(%rsp) ;					\
+	movq	%r10,TF_R10(%rsp) ;					\
+	movq	%r11,TF_R11(%rsp) ;					\
+	movq	%r12,TF_R12(%rsp) ;					\
+	movq	%r13,TF_R13(%rsp) ;					\
+	movq	%r14,TF_R14(%rsp) ;					\
+	movq	%r15,TF_R15(%rsp) ;					\
 	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX late to avoid double count */ \
-	pushl	$irq_num; 	/* pass the IRQ */			\
+	movq	$irq_num, %rdi; 	/* pass the IRQ */		\
 	call	atpic_handle_intr ;					\
-	addl	$4, %esp ;	/* discard the parameter */		\
-;									\
 	MEXITCOUNT ;							\
 	jmp	doreti
 
diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c
index a11afb4..a2e751e 100644
--- a/sys/amd64/isa/clock.c
+++ b/sys/amd64/isa/clock.c
@@ -69,8 +69,12 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/clock.h>
 #include <machine/frame.h>
+#include <machine/intr_machdep.h>
 #include <machine/md_var.h>
 #include <machine/psl.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 #include <machine/specialreg.h>
 
 #include <amd64/isa/icu.h>
@@ -81,8 +85,6 @@ __FBSDID("$FreeBSD$");
 #endif
 #include <amd64/isa/timerreg.h>
 
-#include <amd64/isa/intr_machdep.h>
-
 /*
  * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
  * can use a simple formula for leap years.
@@ -112,6 +114,7 @@ static	u_int	hardclock_max_count;
 static	u_int32_t i8254_lastcount;
 static	u_int32_t i8254_offset;
 static	int	i8254_ticked;
+static	struct intsrc *i8254_intsrc;
 static	u_char	rtc_statusa = RTCSA_DIVIDER | RTCSA_NOPROF;
 static	u_char	rtc_statusb = RTCSB_24HR | RTCSB_PINTR;
 
@@ -122,7 +125,6 @@ static	u_char	rtc_statusb = RTCSB_24HR | RTCSB_PINTR;
 #define	ACQUIRE_PENDING	3
 
 static	u_char	timer2_state;
-static	void	(*timer_func)(struct clockframe *frame) = hardclock;
 
 static	unsigned i8254_get_timecount(struct timecounter *tc);
 static	void	set_timer_freq(u_int freq, int intr_freq);
@@ -137,7 +139,7 @@ static struct timecounter i8254_timecounter = {
 };
 
 static void
-clkintr(struct clockframe frame)
+clkintr(struct clockframe *frame)
 {
 
 	if (timecounter->tc_get_timecount == i8254_get_timecount) {
@@ -151,7 +153,10 @@ clkintr(struct clockframe frame)
 		clkintr_pending = 0;
 		mtx_unlock_spin(&clock_lock);
 	}
-	timer_func(&frame);
+	hardclock(frame);
+#ifdef SMP
+	forward_hardclock();
+#endif
 }
 
 int
@@ -207,16 +212,19 @@ release_timer2()
  * in the statistics, but the stat clock will no longer stop.
  */
 static void
-rtcintr(struct clockframe frame)
+rtcintr(struct clockframe *frame)
 {
 	while (rtcin(RTC_INTR) & RTCIR_PERIOD) {
 		if (profprocs != 0) {
 			if (--pscnt == 0)
 				pscnt = psdiv;
-			profclock(&frame);
+			profclock(frame);
 		}
 		if (pscnt == psdiv)
-			statclock(&frame);
+			statclock(frame);
+#ifdef SMP
+		forward_statclock();
+#endif
 	}
 }
 
@@ -719,7 +727,6 @@ void
 cpu_initclocks()
 {
 	int diag;
-	register_t crit;
 
 	if (statclock_disable) {
 		/*
@@ -735,19 +742,9 @@ cpu_initclocks()
 		profhz = RTC_PROFRATE;
         }
 
-	/* Finish initializing 8253 timer 0. */
-	/*
-	 * XXX Check the priority of this interrupt handler.  I
-	 * couldn't find anything suitable in the BSD/OS code (grog,
-	 * 19 July 2000).
-	 */
-	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
+	/* Finish initializing 8254 timer 0. */
+	intr_add_handler("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(IRQ0);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
 
 	/* Initialize RTC. */
 	writertc(RTC_STATUSA, rtc_statusa);
@@ -760,14 +757,9 @@ cpu_initclocks()
 	if (diag != 0)
 		printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS);
 
-	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
+	intr_add_handler("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(IRQ8);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
+	i8254_intsrc = intr_lookup_source(8);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 
@@ -833,8 +825,8 @@ i8254_get_timecount(struct timecounter *tc)
 	if (count < i8254_lastcount ||
 	    (!i8254_ticked && (clkintr_pending ||
 	    ((count < 20 || (!(rflags & PSL_I) && count < timer0_max_count / 2u)) &&
-	    (inb(IO_ICU1) & 1)))
-	    )) {
+	    i8254_intsrc != NULL &&
+	    i8254_intsrc->is_pic->pic_source_pending(i8254_intsrc))))) {
 		i8254_ticked = 1;
 		i8254_offset += timer0_max_count;
 	}
diff --git a/sys/amd64/isa/icu.h b/sys/amd64/isa/icu.h
deleted file mode 100644
index 06b8955..0000000
--- a/sys/amd64/isa/icu.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/*-
- * Copyright (c) 1990 The Regents of the University of California.
- * All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * William Jolitz.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	from: @(#)icu.h	5.6 (Berkeley) 5/9/91
- * $FreeBSD$
- */
-
-/*
- * AT/386 Interrupt Control constants
- * W. Jolitz 8/89
- */
-
-#ifndef _I386_ISA_ICU_H_
-#define	_I386_ISA_ICU_H_
-
-#ifndef	LOCORE
-
-/*
- * Note:
- *	Most of the SMP equivilants of the icu macros are coded
- *	elsewhere in an MP-safe fashion.
- *	In particular note that the 'imen' variable is opaque.
- *	DO NOT access imen directly, use INTREN()/INTRDIS().
- */
-
-void	INTREN(u_int);
-void	INTRDIS(u_int);
-
-extern	unsigned imen;		/* interrupt mask enable */
-
-#endif /* LOCORE */
-
-
-/*
- * Interrupt enable bits - in normal order of priority (which we change)
- */
-#define	IRQ0		0x0001		/* highest priority - timer */
-#define	IRQ1		0x0002
-#define	IRQ_SLAVE	0x0004
-#define	IRQ8		0x0100
-#define	IRQ9		0x0200
-#define	IRQ2		IRQ9
-#define	IRQ10		0x0400
-#define	IRQ11		0x0800
-#define	IRQ12		0x1000
-#define	IRQ13		0x2000
-#define	IRQ14		0x4000
-#define	IRQ15		0x8000
-#define	IRQ3		0x0008		/* this is highest after rotation */
-#define	IRQ4		0x0010
-#define	IRQ5		0x0020
-#define	IRQ6		0x0040
-#define	IRQ7		0x0080		/* lowest - parallel printer */
-
-/* Initialization control word 1. Written to even address. */
-#define	ICW1_IC4	0x01		/* ICW4 present */
-#define	ICW1_SNGL	0x02		/* 1 = single, 0 = cascaded */
-#define	ICW1_ADI	0x04		/* 1 = 4, 0 = 8 byte vectors */
-#define	ICW1_LTIM	0x08		/* 1 = level trigger, 0 = edge */
-#define	ICW1_RESET	0x10		/* must be 1 */
-/* 0x20 - 0x80 - in 8080/8085 mode only */
-
-/* Initialization control word 2. Written to the odd address. */
-/* No definitions, it is the base vector of the IDT for 8086 mode */
-
-/* Initialization control word 3. Written to the odd address. */
-/* For a master PIC, bitfield indicating a slave 8259 on given input */
-/* For slave, lower 3 bits are the slave's ID binary id on master */
-
-/* Initialization control word 4. Written to the odd address. */
-#define	ICW4_8086	0x01		/* 1 = 8086, 0 = 8080 */
-#define	ICW4_AEOI	0x02		/* 1 = Auto EOI */
-#define	ICW4_MS		0x04		/* 1 = buffered master, 0 = slave */
-#define	ICW4_BUF	0x08		/* 1 = enable buffer mode */
-#define	ICW4_SFNM	0x10		/* 1 = special fully nested mode */
-
-/* Operation control words.  Written after initialization. */
-
-/* Operation control word type 1 */
-/*
- * No definitions.  Written to the odd address.  Bitmask for interrupts.
- * 1 = disabled.
- */
-
-/* Operation control word type 2.  Bit 3 (0x08) must be zero. Even address. */
-#define	OCW2_L0		0x01		/* Level */
-#define	OCW2_L1		0x02
-#define	OCW2_L2		0x04
-/* 0x08 must be 0 to select OCW2 vs OCW3 */
-/* 0x10 must be 0 to select OCW2 vs ICW1 */
-#define	OCW2_EOI	0x20		/* 1 = EOI */
-#define	OCW2_SL		0x40		/* EOI mode */
-#define	OCW2_R		0x80		/* EOI mode */
-
-/* Operation control word type 3.  Bit 3 (0x08) must be set. Even address. */
-#define	OCW3_RIS	0x01
-#define	OCW3_RR		0x02
-#define	OCW3_P		0x04
-/* 0x08 must be 1 to select OCW3 vs OCW2 */
-#define	OCW3_SEL	0x08		/* must be 1 */
-/* 0x10 must be 0 to select OCW3 vs ICW1 */
-#define	OCW3_SMM	0x20		/* special mode mask */
-#define	OCW3_ESMM	0x40		/* enable SMM */
-
-/*
- * Interrupt Control offset into Interrupt descriptor table (IDT)
- */
-#define	ICU_OFFSET	32		/* 0-31 are processor exceptions */
-#define	ICU_LEN		16		/* 32-47 are ISA interrupts */
-#define	HWI_MASK	0xffff		/* bits for h/w interrupts */
-#define	NHWI		16
-
-#define	ICU_IMR_OFFSET	1
-#define	ICU_SLAVEID	2
-#define	ICU_EOI		(OCW2_EOI)	/* non-specific EOI */
-#define	ICU_SETPRI	(OCW2_R | OCW2_SL) /* set rotation priority */
-
-#define	INTRCNT_COUNT	(1 + ICU_LEN + 2 * ICU_LEN)
-
-#endif /* !_I386_ISA_ICU_H_ */
diff --git a/sys/amd64/isa/icu_ipl.S b/sys/amd64/isa/icu_ipl.S
deleted file mode 100644
index ad883fe..0000000
--- a/sys/amd64/isa/icu_ipl.S
+++ /dev/null
@@ -1,81 +0,0 @@
-/*-
- * Copyright (c) 1989, 1990 William F. Jolitz.
- * Copyright (c) 1990 The Regents of the University of California.
- * All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * William Jolitz.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-	.data
-	ALIGN_DATA
-
-/* interrupt mask enable (all h/w off) */
-	.globl	imen
-imen:	.long	HWI_MASK
-
-	.text
-	SUPERALIGN_TEXT
-
-ENTRY(INTREN)
-	movq	%rdi, %rax
-	movl	%eax, %ecx
-	notl	%eax
-	andl	%eax, imen
-	movl	imen, %eax
-	testb	%cl, %cl
-	je	1f
-	outb	%al, $(IO_ICU1 + ICU_IMR_OFFSET)
-1:
-	testb	%ch, %ch
-	je	2f
-	shrl	$8, %eax
-	outb	%al, $(IO_ICU2 + ICU_IMR_OFFSET)
-2:
-	ret
-
-ENTRY(INTRDIS)
-	movq	%rdi, %rax
-	movl	%eax, %ecx
-	orl	%eax, imen
-	movl	imen, %eax
-	testb	%cl, %cl
-	je	1f
-	outb	%al, $(IO_ICU1 + ICU_IMR_OFFSET)
-1:
-	testb	%ch, %ch
-	je	2f
-	shrl	$8, %eax
-	outb	%al, $(IO_ICU2 + ICU_IMR_OFFSET)
-2:
-	ret
diff --git a/sys/amd64/isa/icu_vector.S b/sys/amd64/isa/icu_vector.S
deleted file mode 100644
index 123a731..0000000
--- a/sys/amd64/isa/icu_vector.S
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- *	from: vector.s, 386BSD 0.1 unknown origin
- * $FreeBSD$
- */
-
-#define	IRQ_BIT(irq_num)	(1 << ((irq_num) % 8))
-#define	IRQ_BYTE(irq_num)	((irq_num) >> 3)
-
-#define	ENABLE_ICU1							\
-	movb	$ICU_EOI,%al ;	/* as soon as possible send EOI ... */	\
-	outb	%al,$IO_ICU1	/* ... to clear in service bit */
-
-#define	ENABLE_ICU1_AND_2						\
-	movb	$ICU_EOI,%al ;	/* as above */				\
-	outb	%al,$IO_ICU2 ;	/* but do second icu first ... */	\
-	outb	%al,$IO_ICU1	/* ... then first icu */
-
-
-/*
- * Macros for interrupt interrupt entry, call to handler, and exit.
- */
-
-#define	FAST_INTR(irq_num, vec_name, enable_icus)			\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
-	subq	$TF_RIP,%rsp ;	/* skip dummy tf_err and tf_trapno */	\
-	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
-	jz	1f ;		/* Yes, dont swapgs again */		\
-	swapgs ;							\
-1:	movq	%rdi,TF_RDI(%rsp) ;					\
-	movq	%rsi,TF_RSI(%rsp) ;					\
-	movq	%rdx,TF_RDX(%rsp) ;					\
-	movq	%rcx,TF_RCX(%rsp) ;					\
-	movq	%r8,TF_R8(%rsp) ;					\
-	movq	%r9,TF_R9(%rsp) ;					\
-	movq	%rax,TF_RAX(%rsp) ;					\
-	movq	%rbx,TF_RBX(%rsp) ;					\
-	movq	%rbp,TF_RBP(%rsp) ;					\
-	movq	%r10,TF_R10(%rsp) ;					\
-	movq	%r11,TF_R11(%rsp) ;					\
-	movq	%r12,TF_R12(%rsp) ;					\
-	movq	%r13,TF_R13(%rsp) ;					\
-	movq	%r14,TF_R14(%rsp) ;					\
-	movq	%r15,TF_R15(%rsp) ;					\
-	call	critical_enter ;					\
-	movq	PCPU(CURTHREAD),%rbx ;					\
-	incl	TD_INTR_NESTING_LEVEL(%rbx) ;				\
-	movq	intr_unit + (irq_num) * 8, %rdi ;			\
-	call	*intr_handler + (irq_num) * 8 ;	/* do the work ASAP */	\
-	enable_icus ;		/* (re)enable ASAP (helps edge trigger?) */ \
-	incl	cnt+V_INTR ;	/* book-keeping can wait */		\
-	movq	intr_countp + (irq_num) * 8,%rax ;			\
-	incq	(%rax) ;						\
-	decl	TD_INTR_NESTING_LEVEL(%rbx) ;				\
-	call	critical_exit ;						\
-	jmp	doreti
-
-/* 
- * Slow, threaded interrupts.
- *
- * XXX Most of the parameters here are obsolete.  Fix this when we're
- * done.
- * XXX we really shouldn't return via doreti if we just schedule the
- * interrupt handler and don't run anything.  We could just do an
- * iret.  FIXME.
- */
-#define	INTR(irq_num, vec_name, icu, enable_icus, maybe_extra_ipending) \
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
-	subq	$TF_RIP,%rsp ;	/* skip dummy tf_err and tf_trapno */	\
-	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */	\
-	jz	1f ;		/* Yes, dont swapgs again */		\
-	swapgs ;							\
-1:	movq	%rdi,TF_RDI(%rsp) ;					\
-	movq	%rsi,TF_RSI(%rsp) ;					\
-	movq	%rdx,TF_RDX(%rsp) ;					\
-	movq	%rcx,TF_RCX(%rsp) ;					\
-	movq	%r8,TF_R8(%rsp) ;					\
-	movq	%r9,TF_R9(%rsp) ;					\
-	movq	%rax,TF_RAX(%rsp) ;					\
-	movq	%rbx,TF_RBX(%rsp) ;					\
-	movq	%rbp,TF_RBP(%rsp) ;					\
-	movq	%r10,TF_R10(%rsp) ;					\
-	movq	%r11,TF_R11(%rsp) ;					\
-	movq	%r12,TF_R12(%rsp) ;					\
-	movq	%r13,TF_R13(%rsp) ;					\
-	movq	%r14,TF_R14(%rsp) ;					\
-	movq	%r15,TF_R15(%rsp) ;					\
-	maybe_extra_ipending ;						\
-	movb	imen + IRQ_BYTE(irq_num),%al ;				\
-	orb	$IRQ_BIT(irq_num),%al ;					\
-	movb	%al,imen + IRQ_BYTE(irq_num) ;				\
-	outb	%al,$icu+ICU_IMR_OFFSET ;				\
-	enable_icus ;							\
-	movq	PCPU(CURTHREAD),%rbx ;					\
-	incl	TD_INTR_NESTING_LEVEL(%rbx) ;				\
-	movq	$irq_num, %rdi;	/* pass the IRQ */			\
-	call	sched_ithd ;						\
-	decl	TD_INTR_NESTING_LEVEL(%rbx) ;				\
-	/* We could usually avoid the following jmp by inlining some of */ \
-	/* doreti, but it's probably better to use less cache. */	\
-	jmp	doreti
-
-MCOUNT_LABEL(bintr)
-	FAST_INTR(0,fastintr0, ENABLE_ICU1)
-	FAST_INTR(1,fastintr1, ENABLE_ICU1)
-	FAST_INTR(2,fastintr2, ENABLE_ICU1)
-	FAST_INTR(3,fastintr3, ENABLE_ICU1)
-	FAST_INTR(4,fastintr4, ENABLE_ICU1)
-	FAST_INTR(5,fastintr5, ENABLE_ICU1)
-	FAST_INTR(6,fastintr6, ENABLE_ICU1)
-	FAST_INTR(7,fastintr7, ENABLE_ICU1)
-	FAST_INTR(8,fastintr8, ENABLE_ICU1_AND_2)
-	FAST_INTR(9,fastintr9, ENABLE_ICU1_AND_2)
-	FAST_INTR(10,fastintr10, ENABLE_ICU1_AND_2)
-	FAST_INTR(11,fastintr11, ENABLE_ICU1_AND_2)
-	FAST_INTR(12,fastintr12, ENABLE_ICU1_AND_2)
-	FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2)
-	FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2)
-	FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2)
-
-#define	CLKINTR_PENDING	movl $1,CNAME(clkintr_pending)
-/* Threaded interrupts */
-	INTR(0,intr0, IO_ICU1, ENABLE_ICU1, CLKINTR_PENDING)
-	INTR(1,intr1, IO_ICU1, ENABLE_ICU1,)
-	INTR(2,intr2, IO_ICU1, ENABLE_ICU1,)
-	INTR(3,intr3, IO_ICU1, ENABLE_ICU1,)
-	INTR(4,intr4, IO_ICU1, ENABLE_ICU1,)
-	INTR(5,intr5, IO_ICU1, ENABLE_ICU1,)
-	INTR(6,intr6, IO_ICU1, ENABLE_ICU1,)
-	INTR(7,intr7, IO_ICU1, ENABLE_ICU1,)
-	INTR(8,intr8, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(9,intr9, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(10,intr10, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(11,intr11, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(12,intr12, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2,)
-	INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2,)
-
-MCOUNT_LABEL(eintr)
-
diff --git a/sys/amd64/isa/intr_machdep.c b/sys/amd64/isa/intr_machdep.c
deleted file mode 100644
index 30c12e7..0000000
--- a/sys/amd64/isa/intr_machdep.c
+++ /dev/null
@@ -1,525 +0,0 @@
-/*-
- * Copyright (c) 1991 The Regents of the University of California.
- * All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * William Jolitz.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)isa.c	7.2 (Berkeley) 5/13/91
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include "opt_isa.h"
-
-#include <sys/param.h>
-#include <sys/bus.h> 
-#include <sys/errno.h>
-#include <sys/interrupt.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/lock.h>
-#include <sys/malloc.h>
-#include <sys/module.h>
-#include <sys/mutex.h>
-#include <sys/proc.h>
-#include <sys/syslog.h>
-#include <sys/systm.h>
-#include <sys/unistd.h>
-
-#include <machine/md_var.h>
-#include <machine/segments.h>
-
-#include <amd64/isa/isa.h>
-#include <amd64/isa/icu.h>
-
-#ifdef DEV_ISA
-#include <isa/isavar.h>
-#endif
-#include <amd64/isa/intr_machdep.h>
-#include <sys/interrupt.h>
-
-/*
- * Per-interrupt data.
- */
-u_long	*intr_countp[ICU_LEN];		/* pointers to interrupt counters */
-driver_intr_t	*intr_handler[ICU_LEN];	/* first level interrupt handler */
-struct	 ithd *ithds[ICU_LEN];		/* real interrupt handler */
-void	*intr_unit[ICU_LEN];
-
-static struct	mtx ithds_table_lock;	/* protect the ithds table */
-
-static inthand_t *fastintr[ICU_LEN] = {
-	IDTVEC(fastintr0), IDTVEC(fastintr1),
-	IDTVEC(fastintr2), IDTVEC(fastintr3),
-	IDTVEC(fastintr4), IDTVEC(fastintr5),
-	IDTVEC(fastintr6), IDTVEC(fastintr7),
-	IDTVEC(fastintr8), IDTVEC(fastintr9),
-	IDTVEC(fastintr10), IDTVEC(fastintr11),
-	IDTVEC(fastintr12), IDTVEC(fastintr13),
-	IDTVEC(fastintr14), IDTVEC(fastintr15),
-};
-
-static inthand_t *slowintr[ICU_LEN] = {
-	IDTVEC(intr0), IDTVEC(intr1), IDTVEC(intr2), IDTVEC(intr3),
-	IDTVEC(intr4), IDTVEC(intr5), IDTVEC(intr6), IDTVEC(intr7),
-	IDTVEC(intr8), IDTVEC(intr9), IDTVEC(intr10), IDTVEC(intr11),
-	IDTVEC(intr12), IDTVEC(intr13), IDTVEC(intr14), IDTVEC(intr15),
-};
-
-static driver_intr_t isa_strayintr;
-
-static void	ithds_init(void *dummy);
-static void	ithread_enable(uintptr_t vector);
-static void	ithread_disable(uintptr_t vector);
-static void	init_i8259(void);
-
-#define NMI_PARITY (1 << 7)
-#define NMI_IOCHAN (1 << 6)
-#define ENMI_WATCHDOG (1 << 7)
-#define ENMI_BUSTIMER (1 << 6)
-#define ENMI_IOSTATUS (1 << 5)
-
-#ifdef DEV_ISA
-/*
- * Bus attachment for the ISA PIC.
- */
-static struct isa_pnp_id atpic_ids[] = {
-	{ 0x0000d041 /* PNP0000 */, "AT interrupt controller" },
-	{ 0 }
-};
-
-static int
-atpic_probe(device_t dev)
-{
-	int result;
-	
-	if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, atpic_ids)) <= 0)
-		device_quiet(dev);
-	return(result);
-}
-
-/*
- * The generic ISA attachment code will handle allocating any other resources
- * that we don't explicitly claim here.
- */
-static int
-atpic_attach(device_t dev)
-{
-	return(0);
-}
-
-static device_method_t atpic_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe,		atpic_probe),
-	DEVMETHOD(device_attach,	atpic_attach),
-	DEVMETHOD(device_detach,	bus_generic_detach),
-	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
-	DEVMETHOD(device_suspend,	bus_generic_suspend),
-	DEVMETHOD(device_resume,	bus_generic_resume),
-	{ 0, 0 }
-};
-
-static driver_t atpic_driver = {
-	"atpic",
-	atpic_methods,
-	1,		/* no softc */
-};
-
-static devclass_t atpic_devclass;
-
-DRIVER_MODULE(atpic, isa, atpic_driver, atpic_devclass, 0, 0);
-DRIVER_MODULE(atpic, acpi, atpic_driver, atpic_devclass, 0, 0);
-#endif /* DEV_ISA */
-
-/*
- * Handle a NMI, possibly a machine check.
- * return true to panic system, false to ignore.
- */
-int
-isa_nmi(cd)
-	int cd;
-{
-	int retval = 0;
-	int isa_port = inb(0x61);
-
-	log(LOG_CRIT, "NMI ISA STATUS 0x%02x", isa_port);
-	
-	if (isa_port & NMI_PARITY) {
-		log(LOG_CRIT, "RAM parity error, likely hardware failure.");
-		retval = 1;
-	}
-
-	if (isa_port & NMI_IOCHAN) {
-		log(LOG_CRIT, "I/O channel check, likely hardware failure.");
-		retval = 1;
-	}
-
-	return (retval);
-}
-
-/*
- *  ICU reinitialize when ICU configuration has lost.
- */
-void icu_reinit()
-{
-	int i;
-	register_t crit;
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	init_i8259();
-	for(i=0;i<ICU_LEN;i++)
-		if(intr_handler[i] != isa_strayintr)
-			INTREN(1<<i);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-}
-
-/*
- * Create a default interrupt table to avoid problems caused by
- * spurious interrupts during configuration of kernel, then setup
- * interrupt control unit.
- */
-void
-isa_defaultirq()
-{
-	int i;
-	register_t crit;
-
-	/* icu vectors */
-	for (i = 0; i < ICU_LEN; i++)
-		icu_unset(i, (driver_intr_t *)NULL);
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	init_i8259();
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-}
-
-
-/* 
- *initialize 8259's
- */
-static void init_i8259()
-{
-
-	outb(IO_ICU1, ICW1_RESET | ICW1_IC4);	/* reset; program device, four bytes */
-
-	outb(IO_ICU1+ICU_IMR_OFFSET, NRSVIDT);	/* starting at this vector index */
-	outb(IO_ICU1+ICU_IMR_OFFSET, IRQ_SLAVE);/* slave on line 2 */
-	outb(IO_ICU1+ICU_IMR_OFFSET, ICW4_8086);/* 8086 mode */
-	outb(IO_ICU1+ICU_IMR_OFFSET, 0xff);	/* leave interrupts masked */
-	outb(IO_ICU1, OCW3_SEL | OCW3_RR);	/* default to IRR on read */
-	outb(IO_ICU1, ICU_SETPRI | 0x2);/* pri order 3-7, 0-2 (com2 first) */
-
-	outb(IO_ICU2, ICW1_RESET | ICW1_IC4);	/* reset; program device, four bytes */
-
-	outb(IO_ICU2+ICU_IMR_OFFSET, NRSVIDT+8); /* staring at this vector index */
-	outb(IO_ICU2+ICU_IMR_OFFSET, ICU_SLAVEID); /* my slave id is 2 */
-	outb(IO_ICU2+ICU_IMR_OFFSET, ICW4_8086); /* 8086 mode */
-	outb(IO_ICU2+ICU_IMR_OFFSET, 0xff);     /* leave interrupts masked */
-	outb(IO_ICU2, OCW3_SEL | OCW3_RR);	/* default to IRR on read */
-}
-
-/*
- * Caught a stray interrupt, notify
- */
-static int isaglitch7;
-static int isaglitch15;
-
-static void
-isa_strayintr(vcookiep)
-	void *vcookiep;
-{
-	int intr = (void **)vcookiep - &intr_unit[0];
-	int isr;
-
-	/* Determine if it is a stray interrupt or simply a glitch */
-	if (intr == 7) {
-		outb(IO_ICU1, OCW3_SEL);	/* select IS register */
-		isr = inb(IO_ICU1);
-		outb(IO_ICU1, OCW3_SEL | OCW3_RR | OCW3_RIS); /* reselect IIR */
-		if ((isr & 0x80) == 0) {
-			isaglitch7++;
-			return;
-		}
-	}
-	if (intr == 15) {
-		outb(IO_ICU2, OCW3_SEL);	/* select IS register */
-		isr = inb(IO_ICU2);
-		outb(IO_ICU2, OCW3_SEL | OCW3_RR); /* reselect IIR */
-		if ((isr & 0x80) == 0) {
-			isaglitch15++;
-			return;
-		}
-	}
-	if (intrcnt[1 + intr] <= 5)
-		log(LOG_ERR, "stray irq %d\n", intr);
-	if (intrcnt[1 + intr] == 5)
-		log(LOG_CRIT,
-		    "too many stray irq %d's; not logging any more\n", intr);
-}
-
-#ifdef DEV_ISA
-/*
- * Return a bitmap of the current interrupt requests.  This is 8259-specific
- * and is only suitable for use at probe time.
- */
-intrmask_t
-isa_irq_pending()
-{
-	u_char irr1;
-	u_char irr2;
-
-	irr1 = inb(IO_ICU1);
-	irr2 = inb(IO_ICU2);
-	return ((irr2 << 8) | irr1);
-}
-#endif
-
-/*
- * Update intrnames array with the specified name.  This is used by
- * vmstat(8) and the like.
- */
-static void
-update_intrname(int intr, const char *name)
-{
-	char buf[32];
-	char *cp;
-	int name_index, off, strayintr;
-
-	/*
-	 * Initialise strings for bitbucket and stray interrupt counters.
-	 * These have statically allocated indices 0 and 1 through ICU_LEN.
-	 */
-	if (intrnames[0] == '\0') {
-		off = sprintf(intrnames, "???") + 1;
-		for (strayintr = 0; strayintr < ICU_LEN; strayintr++)
-			off += sprintf(intrnames + off, "stray irq%d",
-			    strayintr) + 1;
-	}
-
-	if (name == NULL)
-		name = "???";
-	if (snprintf(buf, sizeof(buf), "%s irq%d", name, intr) >= sizeof(buf))
-		goto use_bitbucket;
-
-	/*
-	 * Search for `buf' in `intrnames'.  In the usual case when it is
-	 * not found, append it to the end if there is enough space (the \0
-	 * terminator for the previous string, if any, becomes a separator).
-	 */
-	for (cp = intrnames, name_index = 0;
-	    cp != eintrnames && name_index < NR_INTRNAMES;
-	    cp += strlen(cp) + 1, name_index++) {
-		if (*cp == '\0') {
-			if (strlen(buf) >= eintrnames - cp)
-				break;
-			strcpy(cp, buf);
-			goto found;
-		}
-		if (strcmp(cp, buf) == 0)
-			goto found;
-	}
-
-use_bitbucket:
-	printf("update_intrname: counting %s irq%d as %s\n", name, intr,
-	    intrnames);
-	name_index = 0;
-found:
-	intr_countp[intr] = &intrcnt[name_index];
-}
-
-int
-icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
-{
-	register_t	crit;
-
-	if ((u_int)intr >= ICU_LEN || intr == ICU_SLAVEID)
-		return (EINVAL);
-#if 0
-	if (intr_handler[intr] != isa_strayintr)
-		return (EBUSY);
-#endif
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	intr_handler[intr] = handler;
-	intr_unit[intr] = arg;
-	setidt(ICU_OFFSET + intr,
-	       flags & INTR_FAST ? fastintr[intr] : slowintr[intr],
-	       SDT_SYSIGT, SEL_KPL, 0);
-	INTREN(1 << intr);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-	return (0);
-}
-
-/*
- * Dissociate an interrupt handler from an IRQ and set the handler to
- * the stray interrupt handler.  The 'handler' parameter is used only
- * for consistency checking.
- */
-int
-icu_unset(intr, handler)
-	int	intr;
-	driver_intr_t *handler;
-{
-	register_t crit;
-
-	if ((u_int)intr >= ICU_LEN || handler != intr_handler[intr])
-		return (EINVAL);
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTRDIS(1 << intr);
-	intr_countp[intr] = &intrcnt[1 + intr];
-	intr_handler[intr] = isa_strayintr;
-	intr_unit[intr] = &intr_unit[intr];
-	setidt(ICU_OFFSET + intr, slowintr[intr], SDT_SYSIGT, SEL_KPL, 0);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-	return (0);
-}
-
-static void
-ithds_init(void *dummy)
-{
-
-	mtx_init(&ithds_table_lock, "ithread table lock", NULL, MTX_SPIN);
-}
-SYSINIT(ithds_init, SI_SUB_INTR, SI_ORDER_SECOND, ithds_init, NULL);
-
-static void
-ithread_enable(uintptr_t vector)
-{
-	register_t crit;
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTREN(1 << vector);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-}
-
-static void
-ithread_disable(uintptr_t vector)
-{
-	register_t crit;
-
-	crit = intr_disable();
-	mtx_lock_spin(&icu_lock);
-	INTRDIS(1 << vector);
-	mtx_unlock_spin(&icu_lock);
-	intr_restore(crit);
-}
-
-int
-inthand_add(const char *name, int irq, driver_intr_t handler, void *arg,
-    enum intr_type flags, void **cookiep)
-{
-	struct ithd *ithd;		/* descriptor for the IRQ */
-	int errcode = 0;
-	int created_ithd = 0;
-
-	/*
-	 * Work around a race where more than one CPU may be registering
-	 * handlers on the same IRQ at the same time.
-	 */
-	mtx_lock_spin(&ithds_table_lock);
-	ithd = ithds[irq];
-	mtx_unlock_spin(&ithds_table_lock);
-	if (ithd == NULL) {
-		errcode = ithread_create(&ithd, irq, 0, ithread_disable,
-		    ithread_enable, "irq%d:", irq);
-		if (errcode)
-			return (errcode);
-		mtx_lock_spin(&ithds_table_lock);
-		if (ithds[irq] == NULL) {
-			ithds[irq] = ithd;
-			created_ithd++;
-			mtx_unlock_spin(&ithds_table_lock);
-		} else {
-			struct ithd *orphan;
-
-			orphan = ithd;
-			ithd = ithds[irq];
-			mtx_unlock_spin(&ithds_table_lock);
-			ithread_destroy(orphan);
-		}
-	}
-
-	errcode = ithread_add_handler(ithd, name, handler, arg,
-	    ithread_priority(flags), flags, cookiep);
-	
-	if ((flags & INTR_FAST) == 0 || errcode)
-		/*
-		 * The interrupt process must be in place, but
-		 * not necessarily schedulable, before we
-		 * initialize the ICU, since it may cause an
-		 * immediate interrupt.
-		 */
-		if (icu_setup(irq, sched_ithd, arg, flags) != 0)
-			panic("inthand_add: Can't initialize ICU");
-
-	if (errcode)
-		return (errcode);
-	
-	if (flags & INTR_FAST) {
-		errcode = icu_setup(irq, handler, arg, flags);
-		if (errcode && bootverbose)
-			printf("\tinthand_add(irq%d) failed, result=%d\n", 
-			       irq, errcode);
-		if (errcode)
-			return (errcode);
-	}
-
-	update_intrname(irq, name);
-	return (0);
-}
-
-/*
- * Deactivate and remove linked list the interrupt handler descriptor
- * data connected created by an earlier call of inthand_add(), then
- * adjust the interrupt masks if necessary.
- *
- * Return the memory held by the interrupt handler descriptor data
- * structure to the system.  First ensure the handler is not actively
- * in use.
- */
-int
-inthand_remove(void *cookie)
-{
-
-	return (ithread_remove_handler(cookie));
-}
diff --git a/sys/amd64/isa/intr_machdep.h b/sys/amd64/isa/intr_machdep.h
deleted file mode 100644
index 26f500e..0000000
--- a/sys/amd64/isa/intr_machdep.h
+++ /dev/null
@@ -1,118 +0,0 @@
-/*-
- * Copyright (c) 1991 The Regents of the University of California.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef _I386_ISA_INTR_MACHDEP_H_
-#define	_I386_ISA_INTR_MACHDEP_H_
-
-/*
- * Low level interrupt code.
- */ 
-
-#ifdef _KERNEL
-
-#ifdef LOCORE
-
-/*
- * Protects the IO APIC, 8259 PIC, imen, and apic_imen
- */
-#define ICU_LOCK	MTX_LOCK_SPIN(icu_lock, 0)
-#define ICU_UNLOCK	MTX_UNLOCK_SPIN(icu_lock)
-
-#else /* LOCORE */
-
-/*
- * Type of the first (asm) part of an interrupt handler.
- */
-typedef void inthand_t(u_int cs, u_int ef, u_int esp, u_int ss);
-typedef void unpendhand_t(void);
-
-#define	IDTVEC(name)	__CONCAT(X,name)
-
-extern u_long *intr_countp[];	/* pointers into intrcnt[] */
-extern driver_intr_t *intr_handler[];	/* C entry points of intr handlers */
-extern struct ithd *ithds[];
-extern void *intr_unit[];	/* cookies to pass to intr handlers */
-extern struct mtx icu_lock;
-
-inthand_t
-	IDTVEC(fastintr0), IDTVEC(fastintr1),
-	IDTVEC(fastintr2), IDTVEC(fastintr3),
-	IDTVEC(fastintr4), IDTVEC(fastintr5),
-	IDTVEC(fastintr6), IDTVEC(fastintr7),
-	IDTVEC(fastintr8), IDTVEC(fastintr9),
-	IDTVEC(fastintr10), IDTVEC(fastintr11),
-	IDTVEC(fastintr12), IDTVEC(fastintr13),
-	IDTVEC(fastintr14), IDTVEC(fastintr15);
-inthand_t
-	IDTVEC(intr0), IDTVEC(intr1), IDTVEC(intr2), IDTVEC(intr3),
-	IDTVEC(intr4), IDTVEC(intr5), IDTVEC(intr6), IDTVEC(intr7),
-	IDTVEC(intr8), IDTVEC(intr9), IDTVEC(intr10), IDTVEC(intr11),
-	IDTVEC(intr12), IDTVEC(intr13), IDTVEC(intr14), IDTVEC(intr15);
-unpendhand_t
-	IDTVEC(fastunpend0), IDTVEC(fastunpend1), IDTVEC(fastunpend2),
-	IDTVEC(fastunpend3), IDTVEC(fastunpend4), IDTVEC(fastunpend5),
-	IDTVEC(fastunpend6), IDTVEC(fastunpend7), IDTVEC(fastunpend8),
-	IDTVEC(fastunpend9), IDTVEC(fastunpend10), IDTVEC(fastunpend11),
-	IDTVEC(fastunpend12), IDTVEC(fastunpend13), IDTVEC(fastunpend14),
-	IDTVEC(fastunpend15), IDTVEC(fastunpend16), IDTVEC(fastunpend17),
-	IDTVEC(fastunpend18), IDTVEC(fastunpend19), IDTVEC(fastunpend20),
-	IDTVEC(fastunpend21), IDTVEC(fastunpend22), IDTVEC(fastunpend23),
-	IDTVEC(fastunpend24), IDTVEC(fastunpend25), IDTVEC(fastunpend26),
-	IDTVEC(fastunpend27), IDTVEC(fastunpend28), IDTVEC(fastunpend29),
-	IDTVEC(fastunpend30), IDTVEC(fastunpend31);
-
-#define	NR_INTRNAMES	(1 + ICU_LEN + 2 * ICU_LEN)
-
-void	isa_defaultirq(void);
-int	isa_nmi(int cd);
-int	icu_setup(int intr, driver_intr_t *func, void *arg, int flags);
-int	icu_unset(int intr, driver_intr_t *handler);
-void	icu_reinit(void);
-
-/*
- * WARNING: These are internal functions and not to be used by device drivers!
- * They are subject to change without notice. 
- */
-int	inthand_add(const char *name, int irq, driver_intr_t handler, void *arg,
-	    enum intr_type flags, void **cookiep);
-int	inthand_remove(void *cookie);
-void	sched_ithd(void *dummy);
-void	call_fast_unpend(int irq);
-
-#endif /* LOCORE */
-
-#endif /* _KERNEL */
-
-#endif /* !_I386_ISA_INTR_MACHDEP_H_ */
diff --git a/sys/amd64/isa/ithread.c b/sys/amd64/isa/ithread.c
deleted file mode 100644
index e36e866..0000000
--- a/sys/amd64/isa/ithread.c
+++ /dev/null
@@ -1,115 +0,0 @@
-/*-
- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Berkeley Software Design Inc's name may not be used to endorse or
- *    promote products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	BSDI Id: intr.c,v 1.6.2.5 1999/07/06 19:16:52 cp Exp
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-/* Interrupt thread code. */
-
-#include <sys/param.h>
-#include <sys/bus.h>
-#include <sys/interrupt.h>
-#include <sys/systm.h>
-#include <sys/vmmeter.h>
-
-#include <amd64/isa/icu.h>
-#include <amd64/isa/intr_machdep.h>
-#include <amd64/isa/isa.h>
-
-struct int_entropy {
-	struct proc *p;
-	int irq;
-};
-
-static u_int straycount[ICU_LEN];
-static u_int glitchcount7;
-static u_int glitchcount15;
-
-#define	MAX_STRAY_LOG	5
-
-/*
- * Schedule a heavyweight interrupt process.  This function is called
- * from the interrupt handlers Xintr<num>.
- */
-void
-sched_ithd(void *cookie)
-{
-	int irq = (uintptr_t) cookie;	/* IRQ we're handling */
-	struct ithd *ithd = ithds[irq];	/* and the process that does it */
-	int error, isr;
-
-	/* This used to be in icu_vector.s */
-	/*
-	 * We count software interrupts when we process them.  The
-	 * code here follows previous practice, but there's an
-	 * argument for counting hardware interrupts when they're
-	 * processed too.
-	 */
-	atomic_add_long(intr_countp[irq], 1); /* one more for this IRQ */
-	atomic_add_int(&cnt.v_intr, 1); /* one more global interrupt */
-
-	/*
-	 * Schedule the interrupt thread to run if needed and switch to it
-	 * if we schedule it if !cold.
-	 */
-	error = ithread_schedule(ithd, !cold);
-
-	/*
-	 * Log stray interrupts.
-	 */
-	if (error == EINVAL) {
-		/* Determine if it is a stray interrupt or simply a glitch */
-		if (irq == 7) {
-			outb(IO_ICU1, OCW3_SEL);	/* select IS register */
-			isr = inb(IO_ICU1);
-			outb(IO_ICU1, OCW3_SEL | OCW3_RIS); /* reselect IIR */
-			if ((isr & 0x80) == 0) {
-				glitchcount7++;
-				return;
-			}
-		}
-		if (irq == 15) {
-			outb(IO_ICU2, OCW3_SEL);	/* select IS register */
-			isr = inb(IO_ICU2);
-			outb(IO_ICU2, OCW3_SEL | OCW3_RIS); /* reselect IIR */
-			if ((isr & 0x80) == 0) {
-				glitchcount15++;
-				return;
-			}
-		}
-		if (straycount[irq] < MAX_STRAY_LOG) {
-			printf("stray irq %d\n", irq);
-			if (++straycount[irq] == MAX_STRAY_LOG)
-				printf(
-			    "got %d stray irq %d's: not logging anymore\n",
-				    MAX_STRAY_LOG, irq);
-		}
-	}
-}
diff --git a/sys/amd64/isa/vector.S b/sys/amd64/isa/vector.S
deleted file mode 100644
index 2d7be50..0000000
--- a/sys/amd64/isa/vector.S
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- *	from: vector.s, 386BSD 0.1 unknown origin
- * $FreeBSD$
- */
-
-#include <amd64/isa/icu.h>
-#include <amd64/isa/isa.h>
-#include <amd64/isa/intr_machdep.h>
-
-	.data
-	ALIGN_DATA
-
-/*
- * Interrupt counters and names for export to vmstat(8) and friends.
- *
- * XXX this doesn't really belong here; everything except the labels
- * for the endpointers is almost machine-independent.
- */
-
-	.globl	intrcnt, eintrcnt
-intrcnt:
-	.space	INTRCNT_COUNT * 8
-eintrcnt:
-
-	.globl	intrnames, eintrnames
-intrnames:
-	.space	INTRCNT_COUNT * 32
-eintrnames:
-	.text
-
-/*
- * Macros for interrupt interrupt entry, call to handler, and exit.
- *
- * XXX - the interrupt frame is set up to look like a trap frame.  This is
- * usually a waste of time.  The only interrupt handlers that want a frame
- * are the clock handler (it wants a clock frame), the fpu handler (it's
- * easier to do right all in assembler).  The interrupt return routine
- * needs a trap frame for rare AST's (it could easily convert the frame).
- * The direct costs of setting up a trap frame are two pushl's (error
- * code and trap number), an addl to get rid of these, and pushing and
- * popping the call-saved regs %esi, %edi and %ebp twice,  The indirect
- * costs are making the driver interface nonuniform so unpending of
- * interrupts is more complicated and slower (call_driver(unit) would
- * be easier than ensuring an interrupt frame for all handlers.  Finally,
- * there are some struct copies in the fpu handler and maybe in the clock
- * handler that could be avoided by working more with pointers to frames
- * instead of frames.
- *
- * XXX - should we do a cld on every system entry to avoid the requirement
- * for scattered cld's?
- *
- * Coding notes for *.s:
- *
- * If possible, avoid operations that involve an operand size override.
- * Word-sized operations might be smaller, but the operand size override
- * makes them slower on on 486's and no faster on 386's unless perhaps
- * the instruction pipeline is depleted.  E.g.,
- *
- *	Use movl to seg regs instead of the equivalent but more descriptive
- *	movw - gas generates an irelevant (slower) operand size override.
- *
- *	Use movl to ordinary regs in preference to movw and especially
- *	in preference to movz[bw]l.  Use unsigned (long) variables with the
- *	top bits clear instead of unsigned short variables to provide more
- *	opportunities for movl.
- *
- * If possible, use byte-sized operations.  They are smaller and no slower.
- *
- * Use (%reg) instead of 0(%reg) - gas generates larger code for the latter.
- *
- * If the interrupt frame is made more flexible,  INTR can push %eax first
- * and decide the ipending case with less overhead, e.g., by avoiding
- * loading segregs.
- */
-
-#include "amd64/isa/icu_vector.S"
diff --git a/sys/amd64/pci/pci_bus.c b/sys/amd64/pci/pci_bus.c
index 3f0b9dc..6be70b7 100644
--- a/sys/amd64/pci/pci_bus.c
+++ b/sys/amd64/pci/pci_bus.c
@@ -426,7 +426,7 @@ legacy_pcib_probe(device_t dev)
 
 	if (pci_cfgregopen() == 0)
 		return ENXIO;
-	return 0;
+	return -100;
 }
 
 int
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 0af5b67..82ace4c 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -8,48 +8,26 @@
 # dependency lines other than the first are silently ignored.
 #
 
-ia32_genassym.o			optional	ia32			\
+ia32_genassym.o			standard				\
 	dependency 	"$S/compat/ia32/ia32_genassym.c"		\
 	compile-with	"${CC} ${CFLAGS:N-fno-common} -c ${.IMPSRC}"	\
 	no-obj no-implicit-rule						\
 	clean		"ia32_genassym.o"
 #
-ia32_assym.h			optional	ia32			\
+ia32_assym.h			standard				\
 	dependency 	"$S/kern/genassym.sh ia32_genassym.o"		\
 	compile-with	"env NM=${NM} sh $S/kern/genassym.sh ia32_genassym.o > ${.TARGET}" \
 	no-obj no-implicit-rule before-depend				\
 	clean		"ia32_assym.h"
 #
-
-dev/kbd/atkbd.c			optional	atkbd
-dev/kbd/atkbdc.c		optional	atkbdc
-dev/kbd/kbd.c			optional	atkbd
-dev/kbd/kbd.c			optional	kbd
-dev/kbd/kbd.c			optional	sc
-dev/kbd/kbd.c			optional	ukbd
-dev/kbd/kbd.c			optional	vt
-dev/sio/sio.c			optional	sio
-dev/sio/sio_isa.c		optional	sio isa
-dev/syscons/apm/apm_saver.c	optional	apm_saver apm
-dev/syscons/schistory.c		optional	sc
-dev/syscons/scmouse.c		optional	sc
-dev/syscons/scterm.c		optional	sc
-dev/syscons/scterm-dumb.c	optional	sc
-dev/syscons/scterm-sc.c		optional	sc
-dev/syscons/scvesactl.c		optional	sc vga vesa
-dev/syscons/scvgarndr.c		optional	sc vga
-dev/syscons/scvidctl.c		optional	sc
-dev/syscons/scvtb.c		optional	sc
-dev/syscons/syscons.c		optional	sc
-dev/syscons/sysmouse.c		optional	sc
-dev/uart/uart_cpu_amd64.c	optional	uart
-isa/atkbd_isa.c			optional	atkbd
-
 amd64/acpica/OsdEnvironment.c	optional	acpi
 amd64/acpica/acpi_machdep.c	optional	acpi
 amd64/acpica/acpi_wakeup.c	optional	acpi
+amd64/acpica/madt.c		optional	acpi
 amd64/amd64/amd64-gdbstub.c	optional	ddb
 amd64/amd64/amd64_mem.c		standard
+amd64/amd64/apic_vector.S	standard
+amd64/amd64/atomic.c		standard
 amd64/amd64/autoconf.c		standard
 amd64/amd64/busdma_machdep.c	standard
 amd64/amd64/cpu_switch.S	standard
@@ -60,12 +38,21 @@ amd64/amd64/db_trace.c		optional	ddb
 amd64/amd64/dump_machdep.c	standard
 amd64/amd64/elf_machdep.c	standard
 amd64/amd64/exception.S		standard
+amd64/amd64/fpu.c		standard
 amd64/amd64/identcpu.c		standard
+amd64/amd64/in_cksum.c		optional	inet
 amd64/amd64/initcpu.c		standard
+amd64/amd64/intr_machdep.c	standard
+amd64/amd64/io_apic.c		standard
 amd64/amd64/legacy.c		standard
+amd64/amd64/local_apic.c	standard
 amd64/amd64/locore.S		standard	no-obj
 amd64/amd64/machdep.c		standard
 amd64/amd64/mem.c		standard
+amd64/amd64/mp_machdep.c	optional	smp
+amd64/amd64/mpboot.S		optional	smp
+amd64/amd64/mptable.c		optional	mptable
+amd64/amd64/mptable_pci.c	optional	mptable pci
 amd64/amd64/nexus.c		standard
 amd64/amd64/pmap.c		standard
 amd64/amd64/sigtramp.S		standard
@@ -75,72 +62,64 @@ amd64/amd64/trap.c		standard
 amd64/amd64/tsc.c		standard
 amd64/amd64/uma_machdep.c	standard
 amd64/amd64/vm_machdep.c	standard
-amd64/amd64/in_cksum.c		optional inet
-amd64/amd64/fpu.c		standard
-
+amd64/isa/atpic.c		optional	atpic isa
+amd64/isa/atpic_vector.S	optional	atpic isa
 amd64/isa/clock.c		standard
-amd64/isa/intr_machdep.c	standard
 amd64/isa/isa.c			standard
 amd64/isa/isa_dma.c		standard
-amd64/isa/ithread.c		standard
-
-amd64/pci/pci_cfgreg.c		optional	pci
+amd64/isa/nmi.c			standard
 amd64/pci/pci_bus.c		optional	pci
-
-compat/freebsd32/freebsd32_misc.c	optional	ia32
-compat/freebsd32/freebsd32_syscalls.c	optional	ia32
-compat/freebsd32/freebsd32_sysent.c	optional	ia32
-compat/ia32/ia32_sigtramp.S	optional	ia32
-compat/ia32/ia32_sysvec.c	optional	ia32
-amd64/ia32/ia32_signal.c	optional	ia32
-amd64/ia32/ia32_exception.S	optional	ia32
-amd64/ia32/ia32_syscall.c	optional	ia32
-kern/imgact_elf32.c		optional	ia32
-
-# This file tells config what files go into building a kernel,
-# files marked standard are always included.
-#
+amd64/pci/pci_cfgreg.c		optional	pci
 dev/fb/fb.c			optional	fb
 dev/fb/fb.c			optional	vga
 dev/fb/splash.c			optional	splash
 dev/fb/vga.c			optional	vga
-
 dev/kbd/atkbd.c			optional	atkbd
 dev/kbd/atkbdc.c		optional	atkbdc
 dev/kbd/kbd.c			optional	atkbd
 dev/kbd/kbd.c			optional	kbd
 dev/kbd/kbd.c			optional	sc
 dev/kbd/kbd.c			optional	ukbd
-dev/kbd/kbd.c			optional	vt
-
 dev/ppc/ppc.c			optional	ppc
-
+dev/sio/sio.c			optional	sio
+dev/sio/sio_isa.c		optional	sio isa
+dev/syscons/apm/apm_saver.c	optional	apm_saver apm
 dev/syscons/schistory.c		optional	sc
 dev/syscons/scmouse.c		optional	sc
-dev/syscons/scterm.c		optional	sc
 dev/syscons/scterm-dumb.c	optional	sc
 dev/syscons/scterm-sc.c		optional	sc
+dev/syscons/scterm.c		optional	sc
 dev/syscons/scvgarndr.c		optional	sc vga
 dev/syscons/scvidctl.c		optional	sc
 dev/syscons/scvtb.c		optional	sc
 dev/syscons/syscons.c		optional	sc
 dev/syscons/sysmouse.c		optional	sc
+dev/uart/uart_cpu_amd64.c	optional	uart
 geom/geom_bsd.c			standard
 geom/geom_bsd_enc.c		standard
 geom/geom_mbr.c			standard
 geom/geom_mbr_enc.c		standard
-
-#i386/isa/pmtimer.c		optional	pmtimer
-#
 isa/atkbd_isa.c			optional	atkbd
 isa/atkbdc_isa.c		optional	atkbdc
 isa/fd.c			optional	fdc
 isa/psm.c			optional	psm
 isa/syscons_isa.c		optional	sc
 isa/vga_isa.c			optional	vga
-pci/agp_intel.c			optional	agp
-pci/agp_via.c			optional	agp
-pci/agp_sis.c			optional	agp
 pci/agp_ali.c			optional	agp
 pci/agp_amd.c			optional	agp
 pci/agp_i810.c			optional	agp
+pci/agp_intel.c			optional	agp
+pci/agp_sis.c			optional	agp
+pci/agp_via.c			optional	agp
+#
+# IA32 binary support
+#
+amd64/ia32/ia32_exception.S	optional	ia32
+amd64/ia32/ia32_signal.c	optional	ia32
+amd64/ia32/ia32_syscall.c	optional	ia32
+compat/freebsd32/freebsd32_misc.c	optional	ia32
+compat/freebsd32/freebsd32_syscalls.c	optional	ia32
+compat/freebsd32/freebsd32_sysent.c	optional	ia32
+compat/ia32/ia32_sigtramp.S	optional	ia32
+compat/ia32/ia32_sysvec.c	optional	ia32
+kern/imgact_elf32.c		optional	ia32
diff --git a/sys/conf/options.amd64 b/sys/conf/options.amd64
index af1adb7..ccbe0f6 100644
--- a/sys/conf/options.amd64
+++ b/sys/conf/options.amd64
@@ -1,6 +1,8 @@
 # $FreeBSD$
 # Options specific to AMD64 platform kernels
 
+AUTO_EOI_1		opt_auto_eoi.h
+AUTO_EOI_2		opt_auto_eoi.h
 MAXMEM
 PERFMON			opt_perfmon.h
 
@@ -29,3 +31,5 @@ PSM_HOOKRESUME		opt_psm.h
 PSM_RESETAFTERSUSPEND	opt_psm.h
 PSM_DEBUG		opt_psm.h
 IA32
+NO_MIXED_MODE
+DEV_ATPIC		opt_atpic.h
diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index 0190b2b..28c3224 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -816,7 +816,7 @@ pci_add_resources(device_t pcib, device_t bus, device_t dev)
 	}
 
 	if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) {
-#if defined(__ia64__) || defined(__i386__)
+#if defined(__ia64__) || defined(__i386__) || defined(__amd64__)
 		/*
 		 * Try to re-route interrupts. Sometimes the BIOS or
 		 * firmware may leave bogus values in these registers.
diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c
index 3a71f6e..2dee8e3 100644
--- a/sys/kern/kern_switch.c
+++ b/sys/kern/kern_switch.c
@@ -97,7 +97,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/queue.h>
 #include <sys/sched.h>
-#if defined(SMP) && defined(__i386__)
+#if defined(SMP) && (defined(__i386__) || defined(__amd64__))
 #include <sys/smp.h>
 #endif
 #include <machine/critical.h>
@@ -124,7 +124,7 @@ choosethread(void)
 	struct thread *td;
 	struct ksegrp *kg;
 
-#if defined(SMP) && defined(__i386__)
+#if defined(SMP) && (defined(__i386__) || defined(__amd64__))
 	if (smp_active == 0 && PCPU_GET(cpuid) != 0) {
 		/* Shutting down, run idlethread on AP's */
 		td = PCPU_GET(idlethread);
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c
index 6cb5dba..28d5605 100644
--- a/sys/kern/subr_witness.c
+++ b/sys/kern/subr_witness.c
@@ -296,7 +296,7 @@ static struct witness_order_list_entry order_lists[] = {
 	{ "icu", &lock_class_mtx_spin },
 #ifdef SMP
 	{ "smp rendezvous", &lock_class_mtx_spin },
-#ifdef __i386__
+#if defined(__i386__) || defined(__amd64__)
 	{ "tlb", &lock_class_mtx_spin },
 	{ "lazypmap", &lock_class_mtx_spin },
 #endif