Revamp of the syscall path, exception and context handling. The

prime objectives are:
o  Implement a syscall path based on the epc inststruction (see
   sys/ia64/ia64/syscall.s).
o  Revisit the places were we need to save and restore registers
   and define those contexts in terms of the register sets (see
   sys/ia64/include/_regset.h).

Secundairy objectives:
o  Remove the requirement to use contigmalloc for kernel stacks.
o  Better handling of the high FP registers for SMP systems.
o  Switch to the new cpu_switch() and cpu_throw() semantics.
o  Add a good unwinder to reconstruct contexts for the rare
   cases we need to (see sys/contrib/ia64/libuwx)

Many files are affected by this change. Functionally it boils
down to:
o  The EPC syscall doesn't preserve registers it does not need
   to preserve and places the arguments differently on the stack.
   This affects libc and truss.
o  The address of the kernel page directory (kptdir) had to
   be unstaticized for use by the nested TLB fault handler.
   The name has been changed to ia64_kptdir to avoid conflicts.
   The renaming affects libkvm.
o  The trapframe only contains the special registers and the
   scratch registers. For syscalls using the EPC syscall path
   no scratch registers are saved. This affects all places where
   the trapframe is accessed. Most notably the unaligned access
   handler, the signal delivery code and the debugger.
o  Context switching only partly saves the special registers
   and the preserved registers. This affects cpu_switch() and
   triggered the move to the new semantics, which additionally
   affects cpu_throw().
o  The high FP registers are either in the PCB or on some
   CPU. context switching for them is done lazily. This affects
   trap().
o  The mcontext has room for all registers, but not all of them
   have to be defined in all cases. This mostly affects signal
   delivery code now. The *context syscalls are as of yet still
   unimplemented.

Many details went into the removal of the requirement to use
contigmalloc for kernel stacks. The details are mostly CPU
specific and limited to exception_save() and exception_restore().
The few places where we create, destroy or switch stacks were
mostly simplified by not having to construct physical addresses
and additionally saving the virtual addresses for later use.

Besides more efficient context saving and restoring, which of
course yields a noticable speedup, this also fixes the dreaded
SMP bootup problem as a side-effect. The details of which are
still not fully understood.

This change includes all the necessary backward compatibility
code to have it handle older userland binaries that use the
break instruction for syscalls. Support for break-based syscalls
has been pessimized in favor of a clean implementation. Due to
the overall better performance of the kernel, this will still
be notived as an improvement if it's noticed at all.

Approved by: re@ (jhb)

33 files changed, 3245 insertions, 5788 deletions

diff --git a/sys/ia64/ia32/ia32_signal.c b/sys/ia64/ia32/ia32_signal.c
index 18f4ee2..d9066d4 100644
--- a/sys/ia64/ia32/ia32_signal.c
+++ b/sys/ia64/ia32/ia32_signal.c
@@ -52,6 +52,10 @@
 #include <sys/vnode.h>
 #include <sys/imgact_elf.h>
 
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+
 #include <vm/vm.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_param.h>
@@ -64,8 +68,6 @@
 #include <i386/include/psl.h>
 #include <i386/include/segments.h>
 #include <i386/include/specialreg.h>
-#include <machine/frame.h>
-#include <machine/md_var.h>
 
 static register_t *ia32_copyout_strings(struct image_params *imgp);
 static void ia32_setregs(struct thread *td, u_long entry, u_long stack,
@@ -84,9 +86,9 @@ static char ia32_sigcode[] = {
 	0x50,				/* pushl %eax */
 	0xcd, 0x80,			/* int $0x80 */
 	0xeb, 0xfe,			/* 0: jmp 0b */
-	0, 0, 0, 0
+	0
 };
-static int ia32_szsigcode = sizeof(ia32_sigcode) & ~3;
+static int ia32_szsigcode = sizeof(ia32_sigcode);
 
 struct sysentvec ia32_freebsd_sysvec = {
 	SYS_MAXSYSCALL,
@@ -105,8 +107,8 @@ struct sysentvec ia32_freebsd_sysvec = {
 	"FreeBSD ELF",
 	elf32_coredump,
 	NULL,
-	MINSIGSTKSZ,
-	4096,
+	IA32_MINSIGSTKSZ,
+	IA32_PAGE_SIZE,
 	0,
 	IA32_USRSTACK,
 	IA32_USRSTACK,
@@ -145,8 +147,8 @@ ia32_copyout_strings(struct image_params *imgp)
 	 */
 	arginfo = (struct ia32_ps_strings *)IA32_PS_STRINGS;
 	szsigcode = *(imgp->proc->p_sysent->sv_szsigcode);
-	destp =	(caddr_t)arginfo - szsigcode - SPARE_USRSPACE -
-		roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
+	destp =	(caddr_t)arginfo - szsigcode - IA32_USRSPACE -
+	    roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
 
 	/*
 	 * install sigcode
@@ -185,6 +187,7 @@ ia32_copyout_strings(struct image_params *imgp)
 	/*
 	 * vectp also becomes our initial stack base
 	 */
+	vectp = (void*)((uintptr_t)vectp & ~15);
 	stack_base = vectp;
 
 	stringp = imgp->stringbase;
@@ -237,60 +240,45 @@ ia32_copyout_strings(struct image_params *imgp)
 static void
 ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 {
-	struct trapframe *frame = td->td_frame;
+	struct trapframe *tf = td->td_frame;
 	vm_offset_t gdt, ldt;
 	u_int64_t codesel, datasel, ldtsel;
 	u_int64_t codeseg, dataseg, gdtseg, ldtseg;
 	struct segment_descriptor desc;
 	struct vmspace *vmspace = td->td_proc->p_vmspace;
 
-	/*
-	 * Make sure that we restore the entire trapframe after an
-	 * execve.
-	 */
-	frame->tf_flags &= ~FRAME_SYSCALL;
-
-	bzero(frame->tf_r, sizeof(frame->tf_r));
-	bzero(frame->tf_f, sizeof(frame->tf_f));
-
-	frame->tf_cr_iip = entry;
-	frame->tf_cr_ipsr = (IA64_PSR_IC
-			     | IA64_PSR_I
-			     | IA64_PSR_IT
-			     | IA64_PSR_DT
-			     | IA64_PSR_RT
-			     | IA64_PSR_DFH
-			     | IA64_PSR_IS
-			     | IA64_PSR_BN
-			     | IA64_PSR_CPL_USER);
-	frame->tf_r[FRAME_R12] = stack;
+	exec_setregs(td, entry, stack, ps_strings);
+
+	/* Non-syscall frames are cleared by exec_setregs() */
+	if (tf->tf_flags & FRAME_SYSCALL) {
+		bzero(&tf->tf_scratch, sizeof(tf->tf_scratch));
+		bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp));
+	} else
+		tf->tf_special.ndirty = 0;
+
+	tf->tf_special.psr |= IA64_PSR_IS;
+	tf->tf_special.sp = stack;
+
+	/* Point the RSE backstore to something harmless. */
+	tf->tf_special.bspstore = (IA32_PS_STRINGS - ia32_szsigcode -
+	    IA32_USRSPACE + 15) & ~15;
 
 	codesel = LSEL(LUCODE_SEL, SEL_UPL);
 	datasel = LSEL(LUDATA_SEL, SEL_UPL);
 	ldtsel = GSEL(GLDT_SEL, SEL_UPL);
 
-#if 1
-	frame->tf_r[FRAME_R16] = (datasel << 48) | (datasel << 32)
-		| (datasel << 16) | datasel;
-	frame->tf_r[FRAME_R17] = (ldtsel << 32) | (datasel << 16) | codesel;
-#else
-	frame->tf_r[FRAME_R16] = datasel;
-	frame->tf_r[FRAME_R17] = codesel;
-	frame->tf_r[FRAME_R18] = datasel;
-	frame->tf_r[FRAME_R19] = datasel;
-	frame->tf_r[FRAME_R20] = datasel;
-	frame->tf_r[FRAME_R21] = datasel;
-	frame->tf_r[FRAME_R22] = ldtsel;
-#endif
+	/* Setup ia32 segment registers. */
+	tf->tf_scratch.gr16 = (datasel << 48) | (datasel << 32) |
+	    (datasel << 16) | datasel;
+	tf->tf_scratch.gr17 = (ldtsel << 32) | (datasel << 16) | codesel;
 
 	/*
 	 * Build the GDT and LDT.
 	 */
 	gdt = IA32_USRSTACK;
-	vm_map_find(&vmspace->vm_map, 0, 0,
-		    &gdt, PAGE_SIZE, 0,
-		    VM_PROT_ALL, VM_PROT_ALL, 0);
-	ldt = gdt + 4096;
+	vm_map_find(&vmspace->vm_map, 0, 0, &gdt, IA32_PAGE_SIZE << 1, 0,
+	    VM_PROT_ALL, VM_PROT_ALL, 0);
+	ldt = gdt + IA32_PAGE_SIZE;
 
 	desc.sd_lolimit = 8*NLDT-1;
 	desc.sd_lobase = ldt & 0xffffff;
@@ -330,12 +318,13 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (1L << 62) /* 32 bits */
 		+ (1L << 63); /* page granularity */
-	ia64_set_csd(codeseg);
-	ia64_set_ssd(dataseg);
-	frame->tf_r[FRAME_R24] = dataseg; /* ESD */
-	frame->tf_r[FRAME_R27] = dataseg; /* DSD */
-	frame->tf_r[FRAME_R28] = dataseg; /* FSD */
-	frame->tf_r[FRAME_R29] = dataseg; /* GSD */
+
+	tf->tf_scratch.csd = codeseg;
+	tf->tf_scratch.ssd = dataseg;
+	tf->tf_scratch.gr24 = dataseg; /* ESD */
+	tf->tf_scratch.gr27 = dataseg; /* DSD */
+	tf->tf_scratch.gr28 = dataseg; /* FSD */
+	tf->tf_scratch.gr29 = dataseg; /* GSD */
 
 	gdtseg = gdt		/* base */
 		+ ((8L*NGDT - 1) << 32) /* limit */
@@ -351,13 +340,16 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (0L << 62) /* 16 bits */
 		+ (0L << 63); /* byte granularity */
-	frame->tf_r[FRAME_R30] = ldtseg; /* LDTD */
-	frame->tf_r[FRAME_R31] = gdtseg; /* GDTD */
 
+	tf->tf_scratch.gr30 = ldtseg; /* LDTD */
+	tf->tf_scratch.gr31 = gdtseg; /* GDTD */
+
+	/* Set ia32 control registers on this processor. */
+	ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM | CR4_FXSR) << 32));
 	ia64_set_eflag(PSL_USER);
 
 	/* PS_STRINGS value for BSD/OS binaries.  It is 0 for non-BSD/OS. */
-	frame->tf_r[FRAME_R11] = IA32_PS_STRINGS;
+	tf->tf_scratch.gr11 = IA32_PS_STRINGS;
 
 	/*
 	 * XXX - Linux emulator
@@ -366,3 +358,27 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 	 */
 	td->td_retval[1] = 0;
 }
+
+void
+ia32_restorectx(struct pcb *pcb)
+{
+
+	ia64_set_cflg(pcb->pcb_ia32_cflg);
+	ia64_set_eflag(pcb->pcb_ia32_eflag);
+	ia64_set_fcr(pcb->pcb_ia32_fcr);
+	ia64_set_fdr(pcb->pcb_ia32_fdr);
+	ia64_set_fir(pcb->pcb_ia32_fir);
+	ia64_set_fsr(pcb->pcb_ia32_fsr);
+}
+
+void
+ia32_savectx(struct pcb *pcb)
+{
+
+	pcb->pcb_ia32_cflg = ia64_get_cflg();
+	pcb->pcb_ia32_eflag = ia64_get_eflag();
+	pcb->pcb_ia32_fcr = ia64_get_fcr();
+	pcb->pcb_ia32_fdr = ia64_get_fdr();
+	pcb->pcb_ia32_fir = ia64_get_fir();
+	pcb->pcb_ia32_fsr = ia64_get_fsr();
+}
diff --git a/sys/ia64/ia32/ia32_sysvec.c b/sys/ia64/ia32/ia32_sysvec.c
index 18f4ee2..d9066d4 100644
--- a/sys/ia64/ia32/ia32_sysvec.c
+++ b/sys/ia64/ia32/ia32_sysvec.c
@@ -52,6 +52,10 @@
 #include <sys/vnode.h>
 #include <sys/imgact_elf.h>
 
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+
 #include <vm/vm.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_param.h>
@@ -64,8 +68,6 @@
 #include <i386/include/psl.h>
 #include <i386/include/segments.h>
 #include <i386/include/specialreg.h>
-#include <machine/frame.h>
-#include <machine/md_var.h>
 
 static register_t *ia32_copyout_strings(struct image_params *imgp);
 static void ia32_setregs(struct thread *td, u_long entry, u_long stack,
@@ -84,9 +86,9 @@ static char ia32_sigcode[] = {
 	0x50,				/* pushl %eax */
 	0xcd, 0x80,			/* int $0x80 */
 	0xeb, 0xfe,			/* 0: jmp 0b */
-	0, 0, 0, 0
+	0
 };
-static int ia32_szsigcode = sizeof(ia32_sigcode) & ~3;
+static int ia32_szsigcode = sizeof(ia32_sigcode);
 
 struct sysentvec ia32_freebsd_sysvec = {
 	SYS_MAXSYSCALL,
@@ -105,8 +107,8 @@ struct sysentvec ia32_freebsd_sysvec = {
 	"FreeBSD ELF",
 	elf32_coredump,
 	NULL,
-	MINSIGSTKSZ,
-	4096,
+	IA32_MINSIGSTKSZ,
+	IA32_PAGE_SIZE,
 	0,
 	IA32_USRSTACK,
 	IA32_USRSTACK,
@@ -145,8 +147,8 @@ ia32_copyout_strings(struct image_params *imgp)
 	 */
 	arginfo = (struct ia32_ps_strings *)IA32_PS_STRINGS;
 	szsigcode = *(imgp->proc->p_sysent->sv_szsigcode);
-	destp =	(caddr_t)arginfo - szsigcode - SPARE_USRSPACE -
-		roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
+	destp =	(caddr_t)arginfo - szsigcode - IA32_USRSPACE -
+	    roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
 
 	/*
 	 * install sigcode
@@ -185,6 +187,7 @@ ia32_copyout_strings(struct image_params *imgp)
 	/*
 	 * vectp also becomes our initial stack base
 	 */
+	vectp = (void*)((uintptr_t)vectp & ~15);
 	stack_base = vectp;
 
 	stringp = imgp->stringbase;
@@ -237,60 +240,45 @@ ia32_copyout_strings(struct image_params *imgp)
 static void
 ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 {
-	struct trapframe *frame = td->td_frame;
+	struct trapframe *tf = td->td_frame;
 	vm_offset_t gdt, ldt;
 	u_int64_t codesel, datasel, ldtsel;
 	u_int64_t codeseg, dataseg, gdtseg, ldtseg;
 	struct segment_descriptor desc;
 	struct vmspace *vmspace = td->td_proc->p_vmspace;
 
-	/*
-	 * Make sure that we restore the entire trapframe after an
-	 * execve.
-	 */
-	frame->tf_flags &= ~FRAME_SYSCALL;
-
-	bzero(frame->tf_r, sizeof(frame->tf_r));
-	bzero(frame->tf_f, sizeof(frame->tf_f));
-
-	frame->tf_cr_iip = entry;
-	frame->tf_cr_ipsr = (IA64_PSR_IC
-			     | IA64_PSR_I
-			     | IA64_PSR_IT
-			     | IA64_PSR_DT
-			     | IA64_PSR_RT
-			     | IA64_PSR_DFH
-			     | IA64_PSR_IS
-			     | IA64_PSR_BN
-			     | IA64_PSR_CPL_USER);
-	frame->tf_r[FRAME_R12] = stack;
+	exec_setregs(td, entry, stack, ps_strings);
+
+	/* Non-syscall frames are cleared by exec_setregs() */
+	if (tf->tf_flags & FRAME_SYSCALL) {
+		bzero(&tf->tf_scratch, sizeof(tf->tf_scratch));
+		bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp));
+	} else
+		tf->tf_special.ndirty = 0;
+
+	tf->tf_special.psr |= IA64_PSR_IS;
+	tf->tf_special.sp = stack;
+
+	/* Point the RSE backstore to something harmless. */
+	tf->tf_special.bspstore = (IA32_PS_STRINGS - ia32_szsigcode -
+	    IA32_USRSPACE + 15) & ~15;
 
 	codesel = LSEL(LUCODE_SEL, SEL_UPL);
 	datasel = LSEL(LUDATA_SEL, SEL_UPL);
 	ldtsel = GSEL(GLDT_SEL, SEL_UPL);
 
-#if 1
-	frame->tf_r[FRAME_R16] = (datasel << 48) | (datasel << 32)
-		| (datasel << 16) | datasel;
-	frame->tf_r[FRAME_R17] = (ldtsel << 32) | (datasel << 16) | codesel;
-#else
-	frame->tf_r[FRAME_R16] = datasel;
-	frame->tf_r[FRAME_R17] = codesel;
-	frame->tf_r[FRAME_R18] = datasel;
-	frame->tf_r[FRAME_R19] = datasel;
-	frame->tf_r[FRAME_R20] = datasel;
-	frame->tf_r[FRAME_R21] = datasel;
-	frame->tf_r[FRAME_R22] = ldtsel;
-#endif
+	/* Setup ia32 segment registers. */
+	tf->tf_scratch.gr16 = (datasel << 48) | (datasel << 32) |
+	    (datasel << 16) | datasel;
+	tf->tf_scratch.gr17 = (ldtsel << 32) | (datasel << 16) | codesel;
 
 	/*
 	 * Build the GDT and LDT.
 	 */
 	gdt = IA32_USRSTACK;
-	vm_map_find(&vmspace->vm_map, 0, 0,
-		    &gdt, PAGE_SIZE, 0,
-		    VM_PROT_ALL, VM_PROT_ALL, 0);
-	ldt = gdt + 4096;
+	vm_map_find(&vmspace->vm_map, 0, 0, &gdt, IA32_PAGE_SIZE << 1, 0,
+	    VM_PROT_ALL, VM_PROT_ALL, 0);
+	ldt = gdt + IA32_PAGE_SIZE;
 
 	desc.sd_lolimit = 8*NLDT-1;
 	desc.sd_lobase = ldt & 0xffffff;
@@ -330,12 +318,13 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (1L << 62) /* 32 bits */
 		+ (1L << 63); /* page granularity */
-	ia64_set_csd(codeseg);
-	ia64_set_ssd(dataseg);
-	frame->tf_r[FRAME_R24] = dataseg; /* ESD */
-	frame->tf_r[FRAME_R27] = dataseg; /* DSD */
-	frame->tf_r[FRAME_R28] = dataseg; /* FSD */
-	frame->tf_r[FRAME_R29] = dataseg; /* GSD */
+
+	tf->tf_scratch.csd = codeseg;
+	tf->tf_scratch.ssd = dataseg;
+	tf->tf_scratch.gr24 = dataseg; /* ESD */
+	tf->tf_scratch.gr27 = dataseg; /* DSD */
+	tf->tf_scratch.gr28 = dataseg; /* FSD */
+	tf->tf_scratch.gr29 = dataseg; /* GSD */
 
 	gdtseg = gdt		/* base */
 		+ ((8L*NGDT - 1) << 32) /* limit */
@@ -351,13 +340,16 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (0L << 62) /* 16 bits */
 		+ (0L << 63); /* byte granularity */
-	frame->tf_r[FRAME_R30] = ldtseg; /* LDTD */
-	frame->tf_r[FRAME_R31] = gdtseg; /* GDTD */
 
+	tf->tf_scratch.gr30 = ldtseg; /* LDTD */
+	tf->tf_scratch.gr31 = gdtseg; /* GDTD */
+
+	/* Set ia32 control registers on this processor. */
+	ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM | CR4_FXSR) << 32));
 	ia64_set_eflag(PSL_USER);
 
 	/* PS_STRINGS value for BSD/OS binaries.  It is 0 for non-BSD/OS. */
-	frame->tf_r[FRAME_R11] = IA32_PS_STRINGS;
+	tf->tf_scratch.gr11 = IA32_PS_STRINGS;
 
 	/*
 	 * XXX - Linux emulator
@@ -366,3 +358,27 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 	 */
 	td->td_retval[1] = 0;
 }
+
+void
+ia32_restorectx(struct pcb *pcb)
+{
+
+	ia64_set_cflg(pcb->pcb_ia32_cflg);
+	ia64_set_eflag(pcb->pcb_ia32_eflag);
+	ia64_set_fcr(pcb->pcb_ia32_fcr);
+	ia64_set_fdr(pcb->pcb_ia32_fdr);
+	ia64_set_fir(pcb->pcb_ia32_fir);
+	ia64_set_fsr(pcb->pcb_ia32_fsr);
+}
+
+void
+ia32_savectx(struct pcb *pcb)
+{
+
+	pcb->pcb_ia32_cflg = ia64_get_cflg();
+	pcb->pcb_ia32_eflag = ia64_get_eflag();
+	pcb->pcb_ia32_fcr = ia64_get_fcr();
+	pcb->pcb_ia32_fdr = ia64_get_fdr();
+	pcb->pcb_ia32_fir = ia64_get_fir();
+	pcb->pcb_ia32_fsr = ia64_get_fsr();
+}
diff --git a/sys/ia64/ia32/ia32_util.h b/sys/ia64/ia32/ia32_util.h
index 23f2aba..a0b21b0 100644
--- a/sys/ia64/ia32/ia32_util.h
+++ b/sys/ia64/ia32/ia32_util.h
@@ -44,8 +44,11 @@ struct ia32_ps_strings {
 	int	ps_nenvstr;	/* the number of environment strings */
 };
 
-#define IA32_USRSTACK	(4L*1024*1024*1024 - PAGE_SIZE)
-#define IA32_PS_STRINGS	(IA32_USRSTACK - sizeof(struct ia32_ps_strings))
+#define	IA32_MINSIGSTKSZ	2048
+#define	IA32_PAGE_SIZE		4096
+#define	IA32_USRSTACK	(2L*1024*1024*1024 - IA32_PAGE_SIZE*2)
+#define	IA32_PS_STRINGS	(IA32_USRSTACK - sizeof(struct ia32_ps_strings))
+#define	IA32_USRSPACE		IA32_PAGE_SIZE
 
 static __inline caddr_t stackgap_init(void);
 static __inline void *stackgap_alloc(caddr_t *, size_t);
@@ -54,7 +57,7 @@ static __inline caddr_t
 stackgap_init()
 {
 #define	szsigcode (*(curproc->p_sysent->sv_szsigcode))
-	return (caddr_t)(((caddr_t)IA32_PS_STRINGS) - szsigcode - SPARE_USRSPACE);
+	return (((caddr_t)IA32_PS_STRINGS) - szsigcode - IA32_USRSPACE);
 #undef szsigcode
 }
 
diff --git a/sys/ia64/ia64/db_interface.c b/sys/ia64/ia64/db_interface.c
index 34ac65d..3ddece0 100644
--- a/sys/ia64/ia64/db_interface.c
+++ b/sys/ia64/ia64/db_interface.c
@@ -51,7 +51,6 @@
 #include <vm/vm.h>
 
 #include <machine/inst.h>
-#include <machine/rse.h>
 #include <machine/db_machdep.h>
 #include <machine/mutex.h>
 
@@ -70,73 +69,61 @@ extern void	gdb_handle_exception(db_regs_t *, int);
 int	db_active;
 db_regs_t ddb_regs;
 
-static u_int64_t zero;
 static int db_get_rse_reg(struct db_variable *vp, db_expr_t *valuep, int op);
-static int db_get_pc_reg(struct db_variable *vp, db_expr_t *valuep, int op);
+static int db_get_ip_reg(struct db_variable *vp, db_expr_t *valuep, int op);
 
 struct db_variable db_regs[] = {
 	/* Misc control/app registers */
-#define DB_MISC_REGS	15	/* make sure this is correct */
-
-	{"pc",		(db_expr_t*) 0,				db_get_pc_reg},
-	{"ip",		(db_expr_t*) &ddb_regs.tf_cr_iip,	FCN_NULL},
-	{"psr",		(db_expr_t*) &ddb_regs.tf_cr_ipsr,	FCN_NULL},
-	{"cr.isr",	(db_expr_t*) &ddb_regs.tf_cr_isr,	FCN_NULL},
-	{"cr.ifa",	(db_expr_t*) &ddb_regs.tf_cr_ifa,	FCN_NULL},
-	{"pr",		(db_expr_t*) &ddb_regs.tf_pr,		FCN_NULL},
-	{"ar.rsc",	(db_expr_t*) &ddb_regs.tf_ar_rsc,	FCN_NULL},
-	{"ar.pfs",	(db_expr_t*) &ddb_regs.tf_ar_pfs,	FCN_NULL},
-	{"cr.ifs",	(db_expr_t*) &ddb_regs.tf_cr_ifs,	FCN_NULL},
-	{"ar.bspstore",	(db_expr_t*) &ddb_regs.tf_ar_bspstore,	FCN_NULL},
-	{"ar.rnat",	(db_expr_t*) &ddb_regs.tf_ar_rnat,	FCN_NULL},
-	{"ndirty",	(db_expr_t*) &ddb_regs.tf_ndirty,	FCN_NULL},
-	{"ar.unat",	(db_expr_t*) &ddb_regs.tf_ar_unat,	FCN_NULL},
-	{"ar.ccv",	(db_expr_t*) &ddb_regs.tf_ar_ccv,	FCN_NULL},
-	{"ar.fpsr",	(db_expr_t*) &ddb_regs.tf_ar_fpsr,	FCN_NULL},
+#define DB_MISC_REGS	13	/* make sure this is correct */
+
+	{"ip",		NULL,			db_get_ip_reg},
+	{"psr",		(db_expr_t*) &ddb_regs.tf_special.psr,	FCN_NULL},
+	{"cr.isr",	(db_expr_t*) &ddb_regs.tf_special.isr,	FCN_NULL},
+	{"cr.ifa",	(db_expr_t*) &ddb_regs.tf_special.ifa,	FCN_NULL},
+	{"pr",		(db_expr_t*) &ddb_regs.tf_special.pr,	FCN_NULL},
+	{"ar.rsc",	(db_expr_t*) &ddb_regs.tf_special.rsc,	FCN_NULL},
+	{"ar.pfs",	(db_expr_t*) &ddb_regs.tf_special.pfs,	FCN_NULL},
+	{"cr.ifs",	(db_expr_t*) &ddb_regs.tf_special.cfm,	FCN_NULL},
+	{"ar.bspstore",	(db_expr_t*) &ddb_regs.tf_special.bspstore, FCN_NULL},
+	{"ndirty",	(db_expr_t*) &ddb_regs.tf_special.ndirty, FCN_NULL},
+	{"ar.rnat",	(db_expr_t*) &ddb_regs.tf_special.rnat,	FCN_NULL},
+	{"ar.unat",	(db_expr_t*) &ddb_regs.tf_special.unat,	FCN_NULL},
+	{"ar.fpsr",	(db_expr_t*) &ddb_regs.tf_special.fpsr,	FCN_NULL},
 
 	/* Branch registers */
-	{"rp",		(db_expr_t*) &ddb_regs.tf_b[0],		FCN_NULL},
-	{"b1",		(db_expr_t*) &ddb_regs.tf_b[1],		FCN_NULL},
-	{"b2",		(db_expr_t*) &ddb_regs.tf_b[2],		FCN_NULL},
-	{"b3",		(db_expr_t*) &ddb_regs.tf_b[3],		FCN_NULL},
-	{"b4",		(db_expr_t*) &ddb_regs.tf_b[4],		FCN_NULL},
-	{"b5",		(db_expr_t*) &ddb_regs.tf_b[5],		FCN_NULL},
-	{"b6",		(db_expr_t*) &ddb_regs.tf_b[6],		FCN_NULL},
-	{"b7",		(db_expr_t*) &ddb_regs.tf_b[7],		FCN_NULL},
+	{"rp",		(db_expr_t*) &ddb_regs.tf_special.rp,	FCN_NULL},
+	/* b1, b2, b3, b4, b5 are preserved */
+	{"b6",		(db_expr_t*) &ddb_regs.tf_scratch.br6,	FCN_NULL},
+	{"b7",		(db_expr_t*) &ddb_regs.tf_scratch.br7,	FCN_NULL},
 
 	/* Static registers */
-	{"r0",		(db_expr_t*) &zero,			FCN_NULL},
-	{"gp",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R1],	FCN_NULL},
-	{"r2",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R2],	FCN_NULL},
-	{"r3",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R3],	FCN_NULL},
-	{"r4",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R4],	FCN_NULL},
-	{"r5",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R5],	FCN_NULL},
-	{"r6",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R6],	FCN_NULL},
-	{"r7",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R7],	FCN_NULL},
-	{"r8",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R8],	FCN_NULL},
-	{"r9",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R9],	FCN_NULL},
-	{"r10",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R10],	FCN_NULL},
-	{"r11",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R11],	FCN_NULL},
-	{"sp",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R12],	FCN_NULL},
-	{"r13",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R13],	FCN_NULL},
-	{"r14",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R14],	FCN_NULL},
-	{"r15",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R15],	FCN_NULL},
-	{"r16",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R16],	FCN_NULL},
-	{"r17",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R17],	FCN_NULL},
-	{"r18",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R18],	FCN_NULL},
-	{"r19",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R19],	FCN_NULL},
-	{"r20",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R20],	FCN_NULL},
-	{"r21",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R21],	FCN_NULL},
-	{"r22",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R22],	FCN_NULL},
-	{"r23",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R23],	FCN_NULL},
-	{"r24",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R24],	FCN_NULL},
-	{"r25",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R25],	FCN_NULL},
-	{"r26",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R26],	FCN_NULL},
-	{"r27",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R27],	FCN_NULL},
-	{"r28",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R28],	FCN_NULL},
-	{"r29",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R29],	FCN_NULL},
-	{"r30",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R30],	FCN_NULL},
-	{"r31",		(db_expr_t*) &ddb_regs.tf_r[FRAME_R31],	FCN_NULL},
+	{"gp",		(db_expr_t*) &ddb_regs.tf_special.gp,	FCN_NULL},
+	{"r2",		(db_expr_t*) &ddb_regs.tf_scratch.gr2,	FCN_NULL},
+	{"r3",		(db_expr_t*) &ddb_regs.tf_scratch.gr3,	FCN_NULL},
+	{"r8",		(db_expr_t*) &ddb_regs.tf_scratch.gr8,	FCN_NULL},
+	{"r9",		(db_expr_t*) &ddb_regs.tf_scratch.gr9,	FCN_NULL},
+	{"r10",		(db_expr_t*) &ddb_regs.tf_scratch.gr10,	FCN_NULL},
+	{"r11",		(db_expr_t*) &ddb_regs.tf_scratch.gr11,	FCN_NULL},
+	{"sp",		(db_expr_t*) &ddb_regs.tf_special.sp,	FCN_NULL},
+	{"tp",		(db_expr_t*) &ddb_regs.tf_special.tp,	FCN_NULL},
+	{"r14",		(db_expr_t*) &ddb_regs.tf_scratch.gr14,	FCN_NULL},
+	{"r15",		(db_expr_t*) &ddb_regs.tf_scratch.gr15,	FCN_NULL},
+	{"r16",		(db_expr_t*) &ddb_regs.tf_scratch.gr16,	FCN_NULL},
+	{"r17",		(db_expr_t*) &ddb_regs.tf_scratch.gr17,	FCN_NULL},
+	{"r18",		(db_expr_t*) &ddb_regs.tf_scratch.gr18,	FCN_NULL},
+	{"r19",		(db_expr_t*) &ddb_regs.tf_scratch.gr19,	FCN_NULL},
+	{"r20",		(db_expr_t*) &ddb_regs.tf_scratch.gr20,	FCN_NULL},
+	{"r21",		(db_expr_t*) &ddb_regs.tf_scratch.gr21,	FCN_NULL},
+	{"r22",		(db_expr_t*) &ddb_regs.tf_scratch.gr22,	FCN_NULL},
+	{"r23",		(db_expr_t*) &ddb_regs.tf_scratch.gr23,	FCN_NULL},
+	{"r24",		(db_expr_t*) &ddb_regs.tf_scratch.gr24,	FCN_NULL},
+	{"r25",		(db_expr_t*) &ddb_regs.tf_scratch.gr25,	FCN_NULL},
+	{"r26",		(db_expr_t*) &ddb_regs.tf_scratch.gr26,	FCN_NULL},
+	{"r27",		(db_expr_t*) &ddb_regs.tf_scratch.gr27,	FCN_NULL},
+	{"r28",		(db_expr_t*) &ddb_regs.tf_scratch.gr28,	FCN_NULL},
+	{"r29",		(db_expr_t*) &ddb_regs.tf_scratch.gr29,	FCN_NULL},
+	{"r30",		(db_expr_t*) &ddb_regs.tf_scratch.gr30,	FCN_NULL},
+	{"r31",		(db_expr_t*) &ddb_regs.tf_scratch.gr31,	FCN_NULL},
 
 	/* Stacked registers */
 	{"r32",		(db_expr_t*) 32,	db_get_rse_reg},
@@ -241,28 +228,32 @@ struct db_variable *db_eregs = db_regs + sizeof(db_regs)/sizeof(db_regs[0]);
 static int
 db_get_rse_reg(struct db_variable *vp, db_expr_t *valuep, int op)
 {
-	int sof = ddb_regs.tf_cr_ifs & 0x7f;
-	int regno = (db_expr_t) vp->valuep;
-	u_int64_t *bsp = (u_int64_t *) (ddb_regs.tf_ar_bspstore + ddb_regs.tf_ndirty);
 	u_int64_t *reg;
+	uint64_t bsp;
+	int nats, regno, sof;
 
-	if (regno - 32 >= sof) {
-		if (op == DB_VAR_GET)
-			*valuep = 0xdeadbeefdeadbeef;
-	} else {
-		bsp = ia64_rse_previous_frame(bsp, sof);
-		reg = ia64_rse_register_address(bsp, regno);
+	bsp = ddb_regs.tf_special.bspstore + ddb_regs.tf_special.ndirty;
+	regno = (db_expr_t)vp->valuep - 32;
+	sof = (int)(ddb_regs.tf_special.cfm & 0x7f);
+	nats = (sof - regno + 63 - ((int)(bsp >> 3) & 0x3f)) / 63;
+
+	reg = (void*)(bsp - ((sof - regno + nats) << 3));
+
+	if (regno < sof) {
 		if (op == DB_VAR_GET)
 			*valuep = *reg;
 		else
 			*reg = *valuep;
+	} else {
+		if (op == DB_VAR_GET)
+			*valuep = 0xdeadbeefdeadbeef;
 	}
 
-	return 0;
+	return (0);
 }
 
 static int
-db_get_pc_reg(struct db_variable *vp, db_expr_t *valuep, int op)
+db_get_ip_reg(struct db_variable *vp, db_expr_t *valuep, int op)
 {
 	/* Read only */
 	if (op == DB_VAR_GET)
@@ -353,8 +344,8 @@ kdb_trap(int vector, struct trapframe *regs)
 	/*
 	 * XXX pretend that registers outside the current frame don't exist.
 	 */
-	db_eregs = db_regs + DB_MISC_REGS + 8 + 32
-		+ (ddb_regs.tf_cr_ifs & 0x7f);
+	db_eregs = db_regs + DB_MISC_REGS + 3 + 27 +
+	    (ddb_regs.tf_special.cfm & 0x7f);
 
 	__asm __volatile("flushrs"); /* so we can look at them */
 
@@ -412,13 +403,13 @@ db_read_bytes(addr, size, data)
 	register size_t	size;
 	register char	*data;
 {
-	register char	*src;
 
 	db_nofault = &db_jmpbuf;
 
-	src = (char *)addr;
-	while (size-- > 0)
-		*data++ = *src++;
+	if (addr < VM_MAX_ADDRESS)
+		copyin((char *)addr, data, size);
+	else
+		bcopy((char *)addr, data, size);
 
 	db_nofault = 0;
 }
@@ -432,13 +423,13 @@ db_write_bytes(addr, size, data)
 	register size_t	size;
 	register char	*data;
 {
-	register char	*dst;
 
 	db_nofault = &db_jmpbuf;
 
-	dst = (char *)addr;
-	while (size-- > 0)
-		*dst++ = *data++;
+	if (addr < VM_MAX_ADDRESS)
+		copyout(data, (char *)addr, size);
+	else
+		bcopy(data, (char *)addr, size);
 
 	db_nofault = 0;
 }
@@ -455,30 +446,36 @@ db_register_value(regs, regno)
 	db_regs_t *regs;
 	int regno;
 {
-
-	if (regno > 127 || regno < 0) {
-		db_printf(" **** STRANGE REGISTER NUMBER %d **** ", regno);
-		return (0);
-	}
+	uint64_t *rsp;
+	uint64_t bsp;
+	int nats, sof;
 
 	if (regno == 0)
 		return (0);
-
-	if (regno < 32) {
-		return (regs->tf_r[regno - 1]);
-	} else {
-		int sof = ddb_regs.tf_cr_ifs & 0x7f;
-		u_int64_t *bsp = (u_int64_t *) (ddb_regs.tf_ar_bspstore + ddb_regs.tf_ndirty);
-		u_int64_t *reg;
-
-		if (regno - 32 >= sof) {
-			return 0xdeadbeefdeadbeef;
-		} else {
-			bsp = ia64_rse_previous_frame(bsp, sof);
-			reg = ia64_rse_register_address(bsp, regno);
-			return *reg;
-		}
+	if (regno == 1)
+		return (regs->tf_special.gp);
+	if (regno >= 2 && regno <= 3)
+		return ((&regs->tf_scratch.gr2)[regno - 2]);
+	if (regno >= 8 && regno <= 11)
+		return ((&regs->tf_scratch.gr8)[regno - 8]);
+	if (regno == 12)
+		return (regs->tf_special.sp);
+	if (regno == 13)
+		return (regs->tf_special.tp);
+	if (regno >= 14 && regno <= 31)
+		return ((&regs->tf_scratch.gr14)[regno - 14]);
+
+	sof = (int)(regs->tf_special.cfm & 0x7f);
+	if (regno >= 32 && regno < sof + 32) {
+		bsp = regs->tf_special.bspstore + regs->tf_special.ndirty;
+		regno -= 32;
+		nats = (sof - regno + 63 - ((int)(bsp >> 3) & 0x3f)) / 63;
+		rsp = (void*)(bsp - ((sof - regno + nats) << 3));
+		return (*rsp);
 	}
+
+	db_printf(" **** STRANGE REGISTER NUMBER %d **** ", regno);
+	return (0);
 }
 
 void
@@ -539,10 +536,10 @@ db_skip_breakpoint(void)
 	/*
 	 * Skip past the break instruction.
 	 */
-	ddb_regs.tf_cr_ipsr += IA64_PSR_RI_1;
-	if ((ddb_regs.tf_cr_ipsr & IA64_PSR_RI) > IA64_PSR_RI_2) {
-		ddb_regs.tf_cr_ipsr &= ~IA64_PSR_RI;
-		ddb_regs.tf_cr_iip += 16;
+	ddb_regs.tf_special.psr += IA64_PSR_RI_1;
+	if ((ddb_regs.tf_special.psr & IA64_PSR_RI) > IA64_PSR_RI_2) {
+		ddb_regs.tf_special.psr &= ~IA64_PSR_RI;
+		ddb_regs.tf_special.iip += 16;
 	}
 }
 
diff --git a/sys/ia64/ia64/db_trace.c b/sys/ia64/ia64/db_trace.c
index 7c79d06..f093561 100644
--- a/sys/ia64/ia64/db_trace.c
+++ b/sys/ia64/ia64/db_trace.c
@@ -31,7 +31,6 @@
 #include <machine/inst.h>
 #include <machine/db_machdep.h>
 #include <machine/unwind.h>
-#include <machine/rse.h>
 #include <machine/vmparam.h>
 
 #include <ddb/ddb.h>
@@ -45,74 +44,61 @@ int  db_md_set_watchpoint(db_expr_t addr, db_expr_t size);
 int  db_md_clr_watchpoint(db_expr_t addr, db_expr_t size);
 void db_md_list_watchpoints(void);
 
-extern char ia64_vector_table[], do_syscall[], do_syscall_end[];
-
 void
-db_stack_trace_cmd(db_expr_t addr, boolean_t have_addr, db_expr_t count, char *modif)
+db_stack_trace_cmd(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+    char *modif)
 {
-	struct ia64_unwind_state *us;
-
-	if (count == -1)
-		count = 65535;
-
-	if (!have_addr) {
-		us = ia64_create_unwind_state(&ddb_regs);
-	} else {
-		return;		/* XXX */
-	}
-
-	if (!us) {
-		db_printf("db_stack_trace_cmd: can't create unwind state\n");
-		return;
-	}
+	struct unw_regstate rs;
+	const char *name;
+	db_expr_t offset;
+	uint64_t bsp, cfm, ip, pfs, reg;
+	c_db_sym_t sym;
+	int args, error, i;
+
+	error = unw_create(&rs, &ddb_regs);
+	while (!error && count--) {
+		error = unw_get_cfm(&rs, &cfm);
+		if (!error)
+			error = unw_get_bsp(&rs, &bsp);
+		if (!error)
+			error = unw_get_ip(&rs, &ip);
+		if (error)
+			break;
 
-	while (count--) {
-		const char *	name;
-		db_expr_t	ip;
-		db_expr_t	offset;
-		c_db_sym_t	sym;
-		int		cfm, sof, sol, nargs, i;
-		u_int64_t	*bsp;
-		u_int64_t	*p, reg;
-
-		ip = ia64_unwind_state_get_ip(us);
-		cfm = ia64_unwind_state_get_cfm(us);
-		bsp = ia64_unwind_state_get_bsp(us);
-		sof = cfm & 0x7f;
-		sol = (cfm >> 7) & 0x7f;
+		args = (cfm >> 7) & 0x7f;
+		if (args > 8)
+			args = 8;
+
+		error = unw_step(&rs);
+		if (!error) {
+			error = unw_get_cfm(&rs, &pfs);
+			if (!error) {
+				i = (pfs & 0x7f) - ((pfs >> 7) & 0x7f);
+				if (args > i)
+					args = i;
+			}
+		}
 
 		sym = db_search_symbol(ip, DB_STGY_ANY, &offset);
 		db_symbol_values(sym, &name, NULL);
-
 		db_printf("%s(", name);
-
-		nargs = sof - sol;
-		if (nargs > 8)
-			nargs = 8;
-		if (bsp >= (u_int64_t *)IA64_RR_BASE(5)) {
-			for (i = 0; i < nargs; i++) {
-				p = ia64_rse_register_address(bsp, 32 + i);
-				db_read_bytes((vm_offset_t) p, sizeof(reg),
-					      (caddr_t) &reg);
+		if (bsp >= IA64_RR_BASE(5)) {
+			for (i = 0; i < args; i++) {
+				if ((bsp & 0x1ff) == 0x1f8)
+					bsp += 8;
+				db_read_bytes(bsp, sizeof(reg), (void*)&reg);
 				if (i > 0)
 					db_printf(", ");
 				db_printf("0x%lx", reg);
+				bsp += 8;
 			}
-		}
+		} else
+			db_printf("...");
 		db_printf(") at ");
 
 		db_printsym(ip, DB_STGY_PROC);
 		db_printf("\n");
-
-		if (ia64_unwind_state_previous_frame(us))
-			break;
-
-		ip = ia64_unwind_state_get_ip(us);
-		if (!ip)
-			break;
 	}
-
-	ia64_free_unwind_state(us);
 }
 
 void
@@ -143,4 +129,3 @@ db_md_list_watchpoints()
 {
 	return;
 }
-
diff --git a/sys/ia64/ia64/elf_machdep.c b/sys/ia64/ia64/elf_machdep.c
index b3c7d90..6cecfb0 100644
--- a/sys/ia64/ia64/elf_machdep.c
+++ b/sys/ia64/ia64/elf_machdep.c
@@ -59,8 +59,8 @@ struct sysentvec elf64_freebsd_sysvec = {
 	NULL,
 	__elfN(freebsd_fixup),
 	sendsig,
-	sigcode,
-	&szsigcode,
+	NULL,		/* sigcode */
+	NULL,		/* &szsigcode */
 	NULL,
 	"FreeBSD ELF64",
 	__elfN(coredump),
@@ -262,7 +262,7 @@ elf_cpu_load_file(linker_file_t lf)
 
 		if (ph->p_type == PT_IA_64_UNWIND) {
 			vaddr = ph->p_vaddr + reloc;
-			ia64_add_unwind_table((vm_offset_t)lf->address, vaddr,
+			unw_table_add((vm_offset_t)lf->address, vaddr,
 			    vaddr + ph->p_memsz);
 		}
 		++ph;
@@ -275,6 +275,6 @@ int
 elf_cpu_unload_file(linker_file_t lf)
 {
 
-	ia64_delete_unwind_table((vm_offset_t)lf->address);
+	unw_table_remove((vm_offset_t)lf->address);
 	return (0);
 }
diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S
index 9e89aaa..947df2d 100644
--- a/sys/ia64/ia64/exception.S
+++ b/sys/ia64/ia64/exception.S
@@ -1,4 +1,5 @@
 /*-
+ * Copyright (c) 2003 Marcel Moolenaar
  * Copyright (c) 2000 Doug Rabson
  * All rights reserved.
  *
@@ -31,12 +32,541 @@
 #include <assym.s>
 
 /*
- * ar.k7 = curthread
- * ar.k6 = ksp
- * ar.k5 = kbsp
- * ar.k4 = pcpup
+ * ar.k7 = kernel memory stack
+ * ar.k6 = kernel register stack
+ * ar.k5 = EPC gateway page
+ * ar.k4 = PCPU data
  */
 
+	.text
+
+/*
+ * exception_save: save interrupted state
+ *
+ * Arguments:
+ *	r16	address of bundle that contains the branch. The
+ *		return address will be the next bundle.
+ *	r17	the value to save as ifa in the trapframe. This
+ *		normally is cr.ifa, but some interruptions set
+ *		set cr.iim and not cr.ifa.
+ *
+ * Returns:
+ *	p15	interrupted from user stack
+ *	p14	interrupted from kernel stack
+ *	p13	interrupted from user backing store
+ *	p12	interrupted from kernel backing store
+ *	p11	interrupts were enabled
+ *	p10	interrupts were disabled
+ */
+ENTRY(exception_save, 0)
+{	.mii
+	mov		r20=ar.unat
+	extr.u		r31=sp,61,3
+	mov		r18=pr
+	;;
+}
+{	.mmi
+	cmp.le		p14,p15=5,r31
+	;;
+(p15)	mov		r23=ar.k7		// kernel memory stack
+(p14)	mov		r23=sp
+	;;
+}
+{	.mii
+	mov		r21=ar.rsc
+	add		r30=-SIZEOF_TRAPFRAME,r23
+	;;
+	dep		r30=0,r30,0,10
+	;;
+}
+{	.mmi
+	mov		ar.rsc=0
+	sub		r19=r23,r30
+	add		r31=8,r30
+	;;
+}
+{	.mlx
+	mov		r22=cr.iip
+	movl		r26=exception_save_restart
+	;;
+}
+
+	/*
+	 * We have a 1KB aligned trapframe, pointed to by sp. If we write
+	 * to the trapframe, we may trigger a data nested TLB fault. By
+	 * aligning the trapframe on a 1KB boundary, we guarantee that if
+	 * we get a data nested TLB fault, it will be on the very first
+	 * write. Since the data nested TLB fault does not preserve any
+	 * state, we have to be careful what we clobber. Consequently, we
+	 * have to be careful what we use here. Below a list of registers
+	 * that are currently alive:
+	 *	r16,r17=arguments
+	 *	r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
+	 *	r26=restart point
+	 *	r30,r31=trapframe pointers
+	 *	p14,p15=memory stack switch
+	 */
+exception_save_restart:
+{	.mmi
+	st8		[r30]=r19,16		// length
+	st8		[r31]=r0,16		// flags
+	add		r19=16,r19
+	;;
+}
+{	.mmi
+	st8.spill	[r30]=sp,16		// sp
+	st8		[r31]=r20,16		// unat
+	sub		sp=r23,r19
+	;;
+}
+{	.mmi
+	mov		r19=ar.rnat
+	mov		r20=ar.bspstore
+	mov		r23=rp
+	;;
+}
+	// r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp
+{	.mmi
+	st8		[r30]=r23,16		// rp
+	st8		[r31]=r18,16		// pr
+	mov		r24=ar.pfs
+	;;
+}
+{	.mmb
+	st8		[r30]=r24,16		// pfs
+	st8		[r31]=r20,16		// bspstore
+	cover
+	;;
+}
+{	.mmi
+	mov		r18=ar.fpsr
+	mov		r23=cr.ipsr
+	extr.u		r24=r20,61,3
+	;;
+}
+	// r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr
+{	.mmi
+	st8		[r30]=r19,16		// rnat
+	st8		[r31]=r0,16		// __spare
+	cmp.le		p12,p13=5,r24
+	;;
+}
+{	.mmi
+	st8.spill	[r30]=r13,16		// tp
+	st8		[r31]=r21,16		// rsc
+	tbit.nz		p11,p10=r23,14		// p11=interrupts enabled
+	;;
+}
+{	.mmi
+	st8		[r30]=r18,16		// fpsr
+(p13)	mov		r20=ar.k6		// kernel register stack
+	nop		0
+	;;
+}
+	// r20=bspstore, r22=iip, r23=ipsr
+{	.mmi
+	st8		[r31]=r23,16		// psr
+	mov		ar.bspstore=r20
+	nop		0
+	;;
+}
+{	.mmi
+	mov		r18=ar.bsp
+	;;
+	mov		r19=cr.ifs
+	sub		r18=r18,r20
+	;;
+}
+{	.mmi
+	st8.spill	[r30]=gp,16		// gp
+	st8		[r31]=r18,16		// ndirty
+	nop		0
+	;;
+}
+	// r19=ifs, r22=iip
+{	.mmi
+	st8		[r30]=r19,16		// cfm
+	st8		[r31]=r22,16		// iip
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r30]=r17		// ifa
+	mov		r18=cr.isr
+	add		r29=16,r30
+	;;
+}
+{	.mmi
+	st8		[r31]=r18		// isr
+	add		r30=8,r29
+	add		r31=16,r29
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r2,16		// r2
+	.mem.offset	8,0
+	st8.spill	[r31]=r3,16		// r3
+	add		r2=9*8,r29
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r8,16		// r8
+	.mem.offset	8,0
+	st8.spill	[r31]=r9,16		// r9
+	add		r3=8,r2
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r10,16		// r10
+	.mem.offset	8,0
+	st8.spill	[r31]=r11,16		// r11
+	add		r8=16,r16
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r14		// r14
+	.mem.offset	8,0
+	st8.spill	[r31]=r15		// r15
+	mov		r9=r29
+}
+{	.mmb
+	mov		r10=ar.csd
+	mov		r11=ar.ssd
+	bsw.1
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r16,16		// r16
+	.mem.offset	8,0
+	st8.spill	[r3]=r17,16		// r17
+	mov		r14=b6
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r18,16		// r18
+	.mem.offset	8,0
+	st8.spill	[r3]=r19,16		// r19
+	mov		r15=b7
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r20,16		// r20
+	.mem.offset	8,0
+	st8.spill	[r3]=r21,16		// r21
+	mov		b7=r8
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r22,16		// r22
+	.mem.offset	8,0
+	st8.spill	[r3]=r23,16		// r23
+	;;
+}
+
+	.mem.offset	0,0
+	st8.spill	[r2]=r24,16		// r24
+	.mem.offset	8,0
+	st8.spill	[r3]=r25,16		// r25
+	;;
+	.mem.offset	0,0
+	st8.spill	[r2]=r26,16		// r26
+	.mem.offset	8,0
+	st8.spill	[r3]=r27,16		// r27
+	;;
+	.mem.offset	0,0
+	st8.spill	[r2]=r28,16		// r28
+	.mem.offset	8,0
+	st8.spill	[r3]=r29,16		// r29
+	;;
+	.mem.offset	0,0
+	st8.spill	[r2]=r30,16		// r30
+	.mem.offset	8,0
+	st8.spill	[r3]=r31,16		// r31
+	;;
+
+{	.mmi
+	st8		[r2]=r14,16		// b6
+	mov		r17=ar.unat
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r3]=r15,16		// b7
+	mov		r16=ar.ccv
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r2]=r16,16		// ccv
+	st8		[r3]=r10,16		// csd
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r2]=r11,24		// ssd
+	st8		[r9]=r17
+	nop		0
+	;;
+}
+
+	stf.spill	[r3]=f6,32		// f6
+	stf.spill	[r2]=f7,32		// f7
+	;;
+	stf.spill	[r3]=f8,32		// f8
+	stf.spill	[r2]=f9,32		// f9
+	;;
+	stf.spill	[r3]=f10,32		// f10
+	stf.spill	[r2]=f11,32		// f11
+	;;
+	stf.spill	[r3]=f12,32		// f12
+	stf.spill	[r2]=f13,32		// f13
+	;;
+	stf.spill	[r3]=f14		// f14
+	stf.spill	[r2]=f15		// f15
+	;;
+{	.mmi
+	mov		ar.rsc=3
+	mov		r13=ar.k4
+	nop		0
+	;;
+}
+{	.mlx
+	ssm		psr.ic|psr.dfh
+	movl		gp=__gp
+	;;
+}
+{	.mfb
+	srlz.d
+	nop		0
+	br.sptk		b7
+	;;
+}
+END(exception_save)
+
+/*
+ * exception_restore:	restore interrupted state
+ *
+ * Arguments:
+ *	sp+16	trapframe pointer
+ */
+ENTRY(exception_restore, 0)
+{	.mmi
+	rsm		psr.i
+	add		sp=16,sp
+	nop		0
+	;;
+}
+{	.mmi
+	add		r3=SIZEOF_TRAPFRAME-32,sp
+	add		r2=SIZEOF_TRAPFRAME-16,sp
+	add		r8=SIZEOF_SPECIAL+16,sp
+	;;
+}
+	// The next load can trap. Let it be...
+	ldf.fill	f15=[r2],-32		// f15
+	ldf.fill	f14=[r3],-32		// f14
+	;;
+	ldf.fill	f13=[r2],-32		// f13
+	ldf.fill	f12=[r3],-32		// f12
+	;;
+	ldf.fill	f11=[r2],-32		// f11
+	ldf.fill	f10=[r3],-32		// f10
+	;;
+	ldf.fill	f9=[r2],-32		// f9
+	ldf.fill	f8=[r3],-32		// f8
+	;;
+	ldf.fill	f7=[r2],-24		// f7
+	ldf.fill	f6=[r3],-16		// f6
+	;;
+
+{	.mmi
+	ld8		r8=[r8]			// unat (after)
+	;;
+	mov		ar.unat=r8
+	nop		0
+	;;
+}
+
+	ld8		r10=[r2],-16		// ssd
+	ld8		r11=[r3],-16		// csd
+	;;
+	mov		ar.ssd=r10
+	mov		ar.csd=r11
+
+	ld8		r14=[r2],-16		// ccv
+	ld8		r15=[r3],-16		// b7
+	;;
+
+{	.mmi
+	mov		ar.ccv=r14
+	ld8		r8=[r2],-16		// b6
+	mov		b7=r15
+	;;
+}
+{	.mmi
+	ld8.fill	r31=[r3],-16		// r31
+	ld8.fill	r30=[r2],-16		// r30
+	mov		b6=r8
+	;;
+}
+
+	ld8.fill	r29=[r3],-16		// r29
+	ld8.fill	r28=[r2],-16		// r28
+	;;
+	ld8.fill	r27=[r3],-16		// r27
+	ld8.fill	r26=[r2],-16		// r26
+	;;
+	ld8.fill	r25=[r3],-16		// r25
+	ld8.fill	r24=[r2],-16		// r24
+	;;
+	ld8.fill	r23=[r3],-16		// r23
+	ld8.fill	r22=[r2],-16		// r22
+	;;
+	ld8.fill	r21=[r3],-16		// r21
+	ld8.fill	r20=[r2],-16		// r20
+	;;
+	ld8.fill	r19=[r3],-16		// r19
+	ld8.fill	r18=[r2],-16		// r18
+	;;
+
+{	.mmb
+	ld8.fill	r17=[r3],-16		// r17
+	ld8.fill	r16=[r2],-16		// r16
+	bsw.0
+	;;
+}
+{	.mmi
+	ld8.fill	r15=[r3],-16		// r15
+	ld8.fill	r14=[r2],-16		// r14
+	add		r31=16,sp
+	;;
+}
+{	.mmi
+	ld8		r16=[sp]		// tf_length
+	ld8.fill	r11=[r3],-16		// r11
+	add		r30=24,sp
+	;;
+}
+{	.mmi
+	ld8.fill	r10=[r2],-16		// r10
+	ld8.fill	r9=[r3],-16		// r9
+	add		r16=r16,sp		// ar.k7
+	;;
+}
+{	.mmi
+	ld8.fill	r8=[r2],-16		// r8
+	ld8.fill	r3=[r3]			// r3
+	;;
+}
+	// We want nested TLB faults from here on...
+	rsm		psr.ic|psr.i
+	ld8.fill	r2=[r2]			// r2
+	nop		0
+	;;
+	srlz.d
+	ld8.fill	sp=[r31],16		// sp
+	nop		0
+	;;
+
+	ld8		r17=[r30],16		// unat
+	ld8		r29=[r31],16		// rp
+	;;
+	ld8		r18=[r30],16		// pr
+	ld8		r28=[r31],16		// pfs
+	mov		rp=r29
+	;;
+	ld8		r20=[r30],24		// bspstore
+	ld8		r21=[r31],24		// rnat
+	mov		ar.pfs=r28
+	;;
+	ld8.fill	r29=[r30],16		// tp
+	ld8		r22=[r31],16		// rsc
+	;;
+{	.mmi
+	ld8		r23=[r30],16		// fpsr
+	ld8		r24=[r31],16		// psr
+	extr.u		r28=r20,61,3
+	;;
+}
+{	.mmi
+	ld8.fill	r1=[r30],16		// gp
+	ld8		r25=[r31],16		// ndirty
+	cmp.le		p14,p15=5,r28
+	;;
+}
+{	.mmb
+	ld8		r26=[r30]		// cfm
+	ld8		r19=[r31]		// ip
+(p14)	br.cond.sptk	1f
+	;;
+}
+{	.mib
+	// Switch register stack
+	alloc		r30=ar.pfs,0,0,0,0	// discard current frame
+	shl		r31=r25,16		// value for ar.rsc
+	nop		0
+	;;
+}
+	// The loadrs can fault if the backing store is not currently
+	// mapped. We assured forward progress by getting everything we
+	// need from the trapframe so that we don't care if the CPU
+	// purges that translation when it needs to insert a new one for
+	// the backing store.
+{	.mmi
+	mov		ar.rsc=r31		// setup for loadrs
+	mov		ar.k7=r16
+	mov		r13=r29
+	;;
+}
+exception_restore_restart:
+{	.mmi
+	mov		r30=ar.bspstore
+	;;
+	loadrs					// load user regs
+	nop		0
+	;;
+}
+{	.mmi
+	mov		r31=ar.bspstore
+	;;
+	mov		ar.bspstore=r20
+	dep		r31=0,r31,0,9
+	;;
+}
+{	.mmb
+	mov		ar.k6=r31
+	mov		ar.rnat=r21
+	nop		0
+	;;
+}
+1:
+{	.mmb
+	mov		ar.unat=r17
+	mov		cr.iip=r19
+	nop		0
+}
+{	.mmi
+	mov		cr.ipsr=r24
+	mov		cr.ifs=r26
+	mov		pr=r18,0x1fffe
+	;;
+}
+{	.mmb
+	mov		ar.rsc=r22
+	mov		ar.fpsr=r23
+	rfi
+	;;
+}
+END(exception_restore)
+
 /*
  * Call exception_save_regs to preserve the interrupted state in a
  * trapframe. Note that we don't use a call instruction because we
@@ -46,20 +576,27 @@
  * which will restore the interrupted state before executing an rfi to
  * resume it.
  */
-#define TRAP(_n_)				\
-1:	mov	r17=ip;;			\
-	add	r17=2f-1b,r17;			\
-	mov	r16=b6;;			\
-	mov	b6=r17;				\
-	br.sptk.few exception_save;		\
-2: (p3)	ssm	psr.i;				\
-	alloc	r15=ar.pfs,0,0,3,0;		\
-	mov	out0=_n_;			\
-	mov	out1=r14;			\
-	mov	out2=sp;;			\
-	add	sp=-16,sp;;			\
-	br.call.sptk.few rp=trap;		\
-3:	br.sptk.many exception_restore
+#define TRAP(_n_, _ifa_)			\
+{	.mib ;					\
+	mov		r17=_ifa_ ;		\
+	mov		r16=ip ;		\
+	br.sptk		exception_save ;	\
+} ;						\
+{	.mmi ;					\
+(p11)	ssm		psr.i ;;		\
+	alloc		r15=ar.pfs,0,0,2,0 ;	\
+	mov		out0=_n_ ;;		\
+} ;						\
+{	.mfb ;					\
+	add		out1=16,sp ;		\
+	nop		0 ;			\
+	br.call.sptk	rp=trap ;		\
+} ;						\
+{	.mfb ;					\
+	nop		0 ;			\
+	nop		0 ;			\
+	br.sptk		exception_restore ;	\
+}
 
 #define	IVT_ENTRY(name, offset)			\
 	.org	ia64_vector_table + offset;	\
@@ -88,7 +625,7 @@ ivt_##name:
 ia64_vector_table:
 
 IVT_ENTRY(VHPT_Translation, 0x0000)
-	TRAP(0)
+	TRAP(0, cr.ifa)
 IVT_END(VHPT_Translation)
 
 IVT_ENTRY(Instruction_TLB, 0x0400)
@@ -103,8 +640,8 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
 	;; 
 	ld8	r21=[r21]		// check VHPT tag
 	;;
-	cmp.ne	p1,p0=r21,r19
-(p1)	br.dpnt.few 1f
+	cmp.ne	p15,p0=r21,r19
+(p15)	br.dpnt.few 1f
 	;;
 	ld8	r21=[r18]		// read pte
 	mov	pr=r17,0x1ffff
@@ -118,15 +655,15 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
 	;;
 	srlz.d				// serialize
 	;;
-2:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+2:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 3f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 3f		// if not, read next in chain
 	;;
 	ld8	r21=[r20],8		// read pte
 	;; 
@@ -165,7 +702,7 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
 	;;
 	srlz.d
 	;; 
-	TRAP(20)			// Page Not Present trap
+	TRAP(20, cr.ifa)		// Page Not Present trap
 IVT_END(Instruction_TLB)
 
 IVT_ENTRY(Data_TLB, 0x0800)
@@ -180,8 +717,8 @@ IVT_ENTRY(Data_TLB, 0x0800)
 	;; 
 	ld8	r21=[r21]		// check VHPT tag
 	;;
-	cmp.ne	p1,p0=r21,r19
-(p1)	br.dpnt.few 1f
+	cmp.ne	p15,p0=r21,r19
+(p15)	br.dpnt.few 1f
 	;;
 	ld8	r21=[r18]		// read pte
 	mov	pr=r17,0x1ffff
@@ -195,15 +732,15 @@ IVT_ENTRY(Data_TLB, 0x0800)
 	;;
 	srlz.d				// serialize
 	;;
-2:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+2:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 3f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 3f		// if not, read next in chain
 	;;
 	ld8	r21=[r20],8		// read pte
 	;; 
@@ -242,7 +779,7 @@ IVT_ENTRY(Data_TLB, 0x0800)
 	;;
 	srlz.d
 	;; 
-	TRAP(20)			// Page Not Present trap
+	TRAP(20, cr.ifa)		// Page Not Present trap
 IVT_END(Data_TLB)
 
 IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
@@ -251,12 +788,12 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
 	;;
 	extr.u	r17=r16,61,3		// get region number
 	;;
-	cmp.ge	p3,p0=5,r17		// RR0-RR5?
-	cmp.eq	p1,p2=7,r17		// RR7->p1, RR6->p2
-(p3)	br.spnt	9f
+	cmp.ge	p13,p0=5,r17		// RR0-RR5?
+	cmp.eq	p15,p14=7,r17		// RR7->p15, RR6->p14
+(p13)	br.spnt	9f
 	;;
-(p1)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
-(p2)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
+(p15)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
+(p14)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
 	;;
 	dep	r16=0,r16,50,14		// clear bits above PPN
 	;;
@@ -267,7 +804,7 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
 	;;
 	rfi
 9:	mov	pr=r18,0x1ffff		// restore predicates
-	TRAP(3)
+	TRAP(3, cr.ifa)
 IVT_END(Alternate_Instruction_TLB)
 
 IVT_ENTRY(Alternate_Data_TLB, 0x1000)
@@ -276,12 +813,12 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000)
 	;;
 	extr.u	r17=r16,61,3		// get region number
 	;;
-	cmp.ge	p3,p0=5,r17		// RR0-RR5?
-	cmp.eq	p1,p2=7,r17		// RR7->p1, RR6->p2
-(p3)	br.spnt	9f
+	cmp.ge	p13,p0=5,r17		// RR0-RR5?
+	cmp.eq	p15,p14=7,r17		// RR7->p15, RR6->p14
+(p13)	br.spnt	9f
 	;;
-(p1)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
-(p2)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
+(p15)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
+(p14)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
 	;;
 	dep	r16=0,r16,50,14		// clear bits above PPN
 	;;
@@ -292,19 +829,92 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000)
 	;;
 	rfi
 9:	mov	pr=r18,0x1ffff		// restore predicates
-	TRAP(4)
+	TRAP(4, cr.ifa)
 IVT_END(Alternate_Data_TLB)
 
 IVT_ENTRY(Data_Nested_TLB, 0x1400)
-	TRAP(5)
+	// See exception_save_restart and exception_restore_restart for the
+	// contexts that may cause a data nested TLB. We can only use the
+	// banked general registers and predicates, but don't use:
+	//	p14 & p15	-	Set in exception save
+	//	r16 & r17	-	Arguments to exception save
+	//	r30		-	Faulting address (modulo page size)
+	// We assume r30 has the virtual addresses that relate to the data
+	// nested TLB fault. The address does not have to be exact, as long
+	// as it's in the same page. We use physical addressing to avoid
+	// double nested faults. Since all virtual addresses we encounter
+	// here are direct mapped region 7 addresses, we have no problem
+	// constructing physical addresses.
+{	.mlx
+	rsm		psr.dt
+	movl		r27=ia64_kptdir
+	;;
+}
+{	.mii
+	srlz.d
+	dep		r27=0,r27,61,3
+	extr.u		r28=r30,PAGE_SHIFT,61-PAGE_SHIFT
+	;;
+}
+{	.mii
+	ld8		r27=[r27]
+	shr.u		r29=r28,PAGE_SHIFT-5	// dir index
+	extr.u		r28=r28,0,PAGE_SHIFT-5	// pte index
+	;;
+}
+{	.mmi
+	shladd		r27=r29,3,r27
+	;;
+	mov		cr.ifa=r30
+	dep		r27=0,r27,61,3
+	;;
+}
+{	.mmi
+	ld8		r27=[r27]
+	mov		r29=rr[r30]
+	shl		r28=r28,5
+	;;
+}
+{	.mii
+	add		r27=r27,r28		// address of pte
+	dep		r29=0,r29,0,2
+	;;
+	dep		r27=0,r27,61,3
+	;;
+}
+{	.mmi
+	ld8		r28=[r27]
+	;;
+	mov		cr.itir=r29
+	or		r28=PTE_D|PTE_A,r28
+	;;
+}
+{	.mlx
+	st8		[r27]=r28
+	movl		r29=exception_save_restart
+	;;
+}
+{	.mmi
+	itc.d		r28
+	;;
+	ssm		psr.dt
+	cmp.eq		p12,p13=r26,r29
+	;;
+}
+{	.mbb
+	srlz.d
+(p12)	br.sptk		exception_save_restart
+(p13)	br.sptk		exception_restore_restart
+	;;
+}
 IVT_END(Data_Nested_TLB)
 
 IVT_ENTRY(Instruction_Key_Miss, 0x1800)
-	TRAP(6)
+	TRAP(6, cr.ifa)
 IVT_END(Instruction_Key_Miss)
 
 IVT_ENTRY(Data_Key_Miss, 0x1c00)
-	TRAP(7)
+	TRAP(7, cr.ifa)
 IVT_END(Data_Key_Miss)
 
 IVT_ENTRY(Dirty_Bit, 0x2000)
@@ -325,15 +935,15 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
 	;;
 	srlz.d				// serialize
 	;;
-1:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+1:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 2f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 2f		// if not, read next in chain
 	;;
 	ld8	r21=[r20]		// read pte
 	mov	r22=PTE_D|PTE_A
@@ -373,7 +983,7 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
 	br.cond.sptk.few 1b		// loop
 
 9:	mov	pr=r17,0x1ffff		// restore predicates
-	TRAP(8)				// die horribly
+	TRAP(8, cr.ifa)			// die horribly
 IVT_END(Dirty_Bit)
 
 IVT_ENTRY(Instruction_Access_Bit, 0x2400)
@@ -394,15 +1004,15 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400)
 	;;
 	srlz.d				// serialize
 	;;
-1:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+1:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 2f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 2f		// if not, read next in chain
 	;;
 	ld8	r21=[r20]		// read pte
 	mov	r22=PTE_A
@@ -442,7 +1052,7 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400)
 	br.cond.sptk.few 1b		// loop
 
 9:	mov	pr=r17,0x1ffff		// restore predicates
-	TRAP(9)
+	TRAP(9, cr.ifa)
 IVT_END(Instruction_Access_Bit)
 
 IVT_ENTRY(Data_Access_Bit, 0x2800)
@@ -463,15 +1073,15 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
 	;;
 	srlz.d				// serialize
 	;;
-1:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+1:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 2f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 2f		// if not, read next in chain
 	;;
 	ld8	r21=[r20]		// read pte
 	mov	r22=PTE_A
@@ -511,1344 +1121,283 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
 	br.cond.sptk.few 1b		// loop
 
 9:	mov	pr=r17,0x1ffff		// restore predicates
-	TRAP(10)
+	TRAP(10, cr.ifa)
 IVT_END(Data_Access_Bit)
 
 IVT_ENTRY(Break_Instruction, 0x2c00)
-	mov	r16=pr			// save pr for a moment
-	mov	r17=cr.iim;;		// read break value
-	mov	r18=0x100000;;		// syscall number
-	cmp.ne	p6,p0=r18,r17;;		// check for syscall
-(p6)	br.dpnt.few 9f
-
-	mov	r17=cr.ipsr;;		// check for user mode
-	extr.u	r17=r17,32,2;;
-	cmp.eq	p6,p0=r0,r17
-(p6)	br.dpnt.few 9f			// trap if kernel mode
-
-	// Note: p6 and p7 are temporaries so we don't need to restore
-	// the value of pr here since the user-mode program assumes
-	// that syscalls only preserve the function-preserved state.
-
-	br.sptk.many	do_syscall
+{	.mib
+	mov		r17=cr.iim
+	mov		r16=ip
+	br.sptk		exception_save
+	;;
+}
+{	.mmi
+	alloc		r15=ar.pfs,0,0,2,0
+	flushrs
+	mov		out0=11
+	;;
+}
+{	.mib
+(p11)	ssm		psr.i
+	add		out1=16,sp
+	br.call.sptk	rp=trap
 	;;
-9:	mov	pr=r16,0x1ffff		// restore pr
-	TRAP(11)
+}
+{	.mfb
+	nop		0
+	nop		0
+	br.sptk		exception_restore
+	;;
+}
 IVT_END(Break_Instruction)
 
 IVT_ENTRY(External_Interrupt, 0x3000)
-	mov	r16=b6			// save user's b6
-1:	mov	r17=ip;;		// construct return address
-	add	r17=2f-1b,r17;;		// for exception_save
-	mov	b6=r17
-	br.sptk.few exception_save	// 'call' exception_save
-
+{	.mib
+	mov		r17=cr.lid	// cr.iim and cr.ifa are undefined.
+	mov		r16=ip
+	br.sptk		exception_save
+	;;
+}
+{	.mmb
 2:	alloc	r14=ar.pfs,0,0,2,0	// make a frame for calling with
-
-	mov	out1=sp;;
-	add	sp=-16,sp;;
+	add	out1=16,sp
+	nop	0
+}
 
 3:	mov	out0=cr.ivr		// find interrupt vector
 	;;
-	cmp.eq	p6,p0=15,out0		// check for spurious vector number
-(p6)	br.dpnt.few exception_restore	// if spurious, we are done
+	cmp.eq	p15,p0=15,out0		// check for spurious vector number
+(p15)	br.dpnt.few exception_restore	// if spurious, we are done
 	;;
 	ssm	psr.i			// re-enable interrupts
-	;;				// now that we are in-progress
-	srlz.d
-	;;
 	br.call.sptk.many rp=interrupt	// call high-level handler
-
+	;;
 	rsm	psr.i			// disable interrupts
 	;;
 	srlz.d
-	;;
 	mov	cr.eoi=r0		// and ack the interrupt
 	;;
 	srlz.d
 	br.sptk.few 3b			// loop for more
+	;;
 IVT_END(External_Interrupt)
 
 IVT_ENTRY(Reserved_3400, 0x3400)
-	TRAP(13)
+	TRAP(13, cr.ifa)
 IVT_END(Reserved_3400)
 
 IVT_ENTRY(Reserved_3800, 0x3800)
-	TRAP(14)
+	TRAP(14, cr.ifa)
 IVT_END(Reserved_3800)
 
 IVT_ENTRY(Reserved_3c00, 0x3c00)
-	TRAP(15)
+	TRAP(15, cr.ifa)
 IVT_END(Reserved_3c00)
 
 IVT_ENTRY(Reserved_4000, 0x4000)
-	TRAP(16)
+	TRAP(16, cr.ifa)
 IVT_END(Reserved_4000)
 
 IVT_ENTRY(Reserved_4400, 0x4400)
-	TRAP(17)
+	TRAP(17, cr.ifa)
 IVT_END(Reserved_4400)
 
 IVT_ENTRY(Reserved_4800, 0x4800)
-	TRAP(18)
+	TRAP(18, cr.ifa)
 IVT_END(Reserved_4800)
 
 IVT_ENTRY(Reserved_4c00, 0x4c00)
-	TRAP(19)
+	TRAP(19, cr.ifa)
 IVT_END(Reserved_4c00)
 
 IVT_ENTRY(Page_Not_Present, 0x5000)
-	TRAP(20)
+	TRAP(20, cr.ifa)
 IVT_END(Page_Not_Present)
 
 IVT_ENTRY(Key_Permission, 0x5100)
-	TRAP(21)
+	TRAP(21, cr.ifa)
 IVT_END(Key_Permission)
 
 IVT_ENTRY(Instruction_Access_Rights, 0x5200)
-	TRAP(22)
+	TRAP(22, cr.ifa)
 IVT_END(Instruction_Access_Rights)
 
 IVT_ENTRY(Data_Access_Rights, 0x5300)
-	TRAP(23)
+	TRAP(23, cr.ifa)
 IVT_END(Data_Access_Rights)
 
 IVT_ENTRY(General_Exception, 0x5400)
-	TRAP(24)
+	TRAP(24, cr.ifa)
 IVT_END(General_Exception)
 
 IVT_ENTRY(Disabled_FP_Register, 0x5500)
-	TRAP(25)
+	TRAP(25, cr.ifa)
 IVT_END(Disabled_FP_Register)
 
 IVT_ENTRY(NaT_Consumption, 0x5600)
-	TRAP(26)
+	TRAP(26, cr.ifa)
 IVT_END(NaT_Consumption)
 
 IVT_ENTRY(Speculation, 0x5700)
-	TRAP(27)
+	TRAP(27, cr.iim)
 IVT_END(Speculation)
 
 IVT_ENTRY(Reserved_5800, 0x5800)
-	TRAP(28)
+	TRAP(28, cr.ifa)
 IVT_END(Reserved_5800)
 
 IVT_ENTRY(Debug, 0x5900)
-	TRAP(29)
+	TRAP(29, cr.ifa)
 IVT_END(Debug)
 
 IVT_ENTRY(Unaligned_Reference, 0x5a00)
-	TRAP(30)
+	TRAP(30, cr.ifa)
 IVT_END(Unaligned_Reference)
 
 IVT_ENTRY(Unsupported_Data_Reference, 0x5b00)
-	TRAP(31)
+	TRAP(31, cr.ifa)
 IVT_END(Unsupported_Data_Reference)
 
 IVT_ENTRY(Floating_Point_Fault, 0x5c00)
-	TRAP(32)
+	TRAP(32, cr.ifa)
 IVT_END(Floating_Point_Fault)
 
 IVT_ENTRY(Floating_Point_Trap, 0x5d00)
-	TRAP(33)
+	TRAP(33, cr.ifa)
 IVT_END(Floating_Point_Trap)
 
 IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00)
-	TRAP(34)
+	TRAP(34, cr.ifa)
 IVT_END(Lower_Privilege_Transfer_Trap)
 
 IVT_ENTRY(Taken_Branch_Trap, 0x5f00)
-	TRAP(35)
+	TRAP(35, cr.ifa)
 IVT_END(Taken_Branch_Trap)
 
 IVT_ENTRY(Single_Step_Trap, 0x6000)
-	TRAP(36)
+	TRAP(36, cr.ifa)
 IVT_END(Single_Step_Trap)
 
 IVT_ENTRY(Reserved_6100, 0x6100)
-	TRAP(37)
+	TRAP(37, cr.ifa)
 IVT_END(Reserved_6100)
 
 IVT_ENTRY(Reserved_6200, 0x6200)
-	TRAP(38)
+	TRAP(38, cr.ifa)
 IVT_END(Reserved_6200)
 
 IVT_ENTRY(Reserved_6300, 0x6300)
-	TRAP(39)
+	TRAP(39, cr.ifa)
 IVT_END(Reserved_6300)
 
 IVT_ENTRY(Reserved_6400, 0x6400)
-	TRAP(40)
+	TRAP(40, cr.ifa)
 IVT_END(Reserved_6400)
 
 IVT_ENTRY(Reserved_6500, 0x6500)
-	TRAP(41)
+	TRAP(41, cr.ifa)
 IVT_END(Reserved_6500)
 
 IVT_ENTRY(Reserved_6600, 0x6600)
-	TRAP(42)
+	TRAP(42, cr.ifa)
 IVT_END(Reserved_6600)
 
 IVT_ENTRY(Reserved_6700, 0x6700)
-	TRAP(43)
+	TRAP(43, cr.ifa)
 IVT_END(Reserved_6700)
 
 IVT_ENTRY(Reserved_6800, 0x6800)
-	TRAP(44)
+	TRAP(44, cr.ifa)
 IVT_END(Reserved_6800)
 
 IVT_ENTRY(IA_32_Exception, 0x6900)
-	TRAP(45)
+	TRAP(45, cr.ifa)
 IVT_END(IA_32_Exception)
 
 IVT_ENTRY(IA_32_Intercept, 0x6a00)
-	TRAP(46)
+	TRAP(46, cr.iim)
 IVT_END(IA_32_Intercept)
 
 IVT_ENTRY(IA_32_Interrupt, 0x6b00)
-	TRAP(47)
+	TRAP(47, cr.ifa)
 IVT_END(IA_32_Interrupt)
 
 IVT_ENTRY(Reserved_6c00, 0x6c00)
-	TRAP(48)
+	TRAP(48, cr.ifa)
 IVT_END(Reserved_6c00)
 
 IVT_ENTRY(Reserved_6d00, 0x6d00)
-	TRAP(49)
+	TRAP(49, cr.ifa)
 IVT_END(Reserved_6d00)
 
 IVT_ENTRY(Reserved_6e00, 0x6e00)
-	TRAP(50)
+	TRAP(50, cr.ifa)
 IVT_END(Reserved_6e00)
 
 IVT_ENTRY(Reserved_6f00, 0x6f00)
-	TRAP(51)
+	TRAP(51, cr.ifa)
 IVT_END(Reserved_6f00)
 
 IVT_ENTRY(Reserved_7000, 0x7000)
-	TRAP(52)
+	TRAP(52, cr.ifa)
 IVT_END(Reserved_7000)
 
 IVT_ENTRY(Reserved_7100, 0x7100)
-	TRAP(53)
+	TRAP(53, cr.ifa)
 IVT_END(Reserved_7100)
 
 IVT_ENTRY(Reserved_7200, 0x7200)
-	TRAP(54)
+	TRAP(54, cr.ifa)
 IVT_END(Reserved_7200)
 
 IVT_ENTRY(Reserved_7300, 0x7300)
-	TRAP(55)
+	TRAP(55, cr.ifa)
 IVT_END(Reserved_7300)
 
 IVT_ENTRY(Reserved_7400, 0x7400)
-	TRAP(56)
+	TRAP(56, cr.ifa)
 IVT_END(Reserved_7400)
 
 IVT_ENTRY(Reserved_7500, 0x7500)
-	TRAP(57)
+	TRAP(57, cr.ifa)
 IVT_END(Reserved_7500)
 
 IVT_ENTRY(Reserved_7600, 0x7600)
-	TRAP(58)
+	TRAP(58, cr.ifa)
 IVT_END(Reserved_7600)
 
 IVT_ENTRY(Reserved_7700, 0x7700)
-	TRAP(59)
+	TRAP(59, cr.ifa)
 IVT_END(Reserved_7700)
 
 IVT_ENTRY(Reserved_7800, 0x7800)
-	TRAP(60)
+	TRAP(60, cr.ifa)
 IVT_END(Reserved_7800)
 
 IVT_ENTRY(Reserved_7900, 0x7900)
-	TRAP(61)
+	TRAP(61, cr.ifa)
 IVT_END(Reserved_7900)
 
 IVT_ENTRY(Reserved_7a00, 0x7a00)
-	TRAP(62)
+	TRAP(62, cr.ifa)
 IVT_END(Reserved_7a00)
 
 IVT_ENTRY(Reserved_7b00, 0x7b00)
-	TRAP(63)
+	TRAP(63, cr.ifa)
 IVT_END(Reserved_7b00)
 
 IVT_ENTRY(Reserved_7c00, 0x7c00)
-	TRAP(64)
+	TRAP(64, cr.ifa)
 IVT_END(Reserved_7c00)
 
 IVT_ENTRY(Reserved_7d00, 0x7d00)
-	TRAP(65)
+	TRAP(65, cr.ifa)
 IVT_END(Reserved_7d00)
 
 IVT_ENTRY(Reserved_7e00, 0x7e00)
-	TRAP(66)
+	TRAP(66, cr.ifa)
 IVT_END(Reserved_7e00)
 
 IVT_ENTRY(Reserved_7f00, 0x7f00)
-	TRAP(67)
+	TRAP(67, cr.ifa)
 IVT_END(Reserved_7f00)
-
-	.text
-
-/*
- * exception_restore:	restore interrupted state
- *
- * Arguments:
- *	sp+16	trapframe pointer
- *	r4	ar.pfs before the alloc in TRAP()
- *
- */
-ENTRY(exception_restore, 0)
-{	.mfi
-	alloc	r14=ar.pfs,0,0,1,0	// in case we call ast()
-	nop	1
-	add	r3=TF_CR_IPSR+16,sp
-	;;
-}
-{	.mmi
-	ld8	r30=[r3]		// ipsr
-	;;
-	nop	2
-	extr.u	r16=r30,32,2		// extract ipsr.cpl
-	;;
-}
-{	.mfb
-	cmp.eq	p1,p2=r0,r16		// test for return to kernel mode
-	nop	3
-(p1)	br.cond.dpnt 2f			// no ast check for returns to kernel
-}
-3:
-{	.mmi
-	add	r3=PC_CURTHREAD,r13	// &curthread
-	;;
-	ld8	r3=[r3]			// curthread
-	add	r2=(TDF_ASTPENDING|TDF_NEEDRESCHED),r0
-	;;
-}
-{	.mmb
-	mov	r15=psr			// save interrupt enable status
-	nop	4
-	;;
-}
-{	.mmi
-	;;
-	rsm	psr.i			// disable interrupts
-	add	r3=TD_FLAGS,r3		// &curthread->td_flags
-	;;
-}
-{	.mmi
-	ld4	r14=[r3]		// fetch curthread->td_flags
-	;;
-	and	r14=r2,r14	    // flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)
-	nop	5
-	;;
-}
-{	.mfb
-	cmp4.eq	p6,p7=r0,r14		//  == 0 ?
-	nop	6
-(p6)	br.cond.dptk	2f
-	;;
-}
-{	.mmi
-	mov	psr.l=r15		// restore interrups
-	;;
-	srlz.d
-	add	out0=16,sp		// trapframe argument to ast()
-}
-{	.bbb
-	br.call.sptk.many rp=ast	// note: p1, p2 preserved
-	br.sptk	3b
-	nop	7
-}
-2:
-{	.mmi
-	rsm	psr.ic|psr.dt|psr.i	// disable interrupt collection and vm
-	;;
-	srlz.i
-	add	r3=16,sp
-	;;
-}
-{	.mmi
-(p2)	add	r16=SIZEOF_TRAPFRAME+16,sp  // restore ar.k6 (kernel sp)
-	;;
-(p2)	mov	ar.k6=r16
-	dep	r3=0,r3,61,3		// physical address
-	;; 
-}
-{	.mmi
-	add	r1=SIZEOF_TRAPFRAME-16,r3 // r1=&tf_f[FRAME_F15]
-	;;
-	ldf.fill f15=[r1],-32		// r1=&tf_f[FRAME_F13]
-	add	r2=SIZEOF_TRAPFRAME-32,r3 // r2=&tf_f[FRAME_F14]
-	;;
-}
-{	.mmb
-	ldf.fill f14=[r2],-32		// r2=&tf_f[FRAME_F12]
-	ldf.fill f13=[r1],-32		// r1=&tf_f[FRAME_F11]
-	nop	8
-	;;
-}
-{	.mmi
-	ldf.fill f12=[r2],-32		// r2=&tf_f[FRAME_F10]
-	ldf.fill f11=[r1],-32		// r1=&tf_f[FRAME_F9]
-	nop	9
-	;;
-}
-{	.mmb
-	ldf.fill f10=[r2],-32		// r2=&tf_f[FRAME_F8]
-	ldf.fill f9=[r1],-32		// r1=&tf_f[FRAME_F7]
-	nop	10
-	;;
-}
-{	.mmi
-	ldf.fill f8=[r2],-32		// r2=&tf_f[FRAME_F6]
-	ldf.fill f7=[r1],-24		// r1=&tf_r[FRAME_R31]
-	nop	11
-	;;
-}
-{	.mmb
-	ldf.fill f6=[r2],-16		// r2=&tf_r[FRAME_R30]
-	ld8.fill r31=[r1],-16		// r1=&tf_r[FRAME_R29]
-	nop	12
-	;;
-}
-{	.mmi
-	ld8.fill r30=[r2],-16		// r2=&tf_r[FRAME_R28]
-	ld8.fill r29=[r1],-16		// r1=&tf_r[FRAME_R27]
-	nop	13
-	;;
-}
-{	.mmb
-	ld8.fill r28=[r2],-16		// r2=&tf_r[FRAME_R26]
-	ld8.fill r27=[r1],-16		// r1=&tf_r[FRAME_R25]
-	nop	14
-	;;
-}
-{	.mmi
-	ld8.fill r26=[r2],-16		// r2=&tf_r[FRAME_R24]
-	ld8.fill r25=[r1],-16		// r1=&tf_r[FRAME_R23]
-	nop	15
-	;;
-}
-{	.mmb
-	ld8.fill r24=[r2],-16		// r2=&tf_r[FRAME_R22]
-	ld8.fill r23=[r1],-16		// r1=&tf_r[FRAME_R21]
-	nop	16
-	;;
-}
-{	.mmi
-	ld8.fill r22=[r2],-16		// r2=&tf_r[FRAME_R20]
-	ld8.fill r21=[r1],-16		// r1=&tf_r[FRAME_R19]
-	nop	17
-	;;
-}
-{	.mmb
-	ld8.fill r20=[r2],-16		// r2=&tf_r[FRAME_R18]
-	ld8.fill r19=[r1],-16		// r1=&tf_r[FRAME_R17]
-	nop	18
-	;;
-}
-{	.mmi
-	ld8.fill r18=[r2],-16		// r2=&tf_r[FRAME_R16]
-	ld8.fill r17=[r1],-16		// r1=&tf_r[FRAME_R15]
-	nop	19
-	;;
-}
-{	.mfb
-	ld8.fill r16=[r2],-16		// r2=&tf_r[FRAME_R14]
-	nop	20
-	bsw.0				// switch to bank 0
-	;;
-}
-{	.mmi
-	ld8.fill r15=[r1],-16		// r1=&tf_r[FRAME_R13]
-	ld8.fill r14=[r2],-16		// r2=&tf_r[FRAME_R12]
-	nop	21
-	;;
-}
-	// Don't restore r13 if returning to kernel
-{	.mmi
-	.pred.rel.mutex p1,p2
-(p2)	ld8.fill r13=[r1],-16		// r1=&tf_r[FRAME_R11]
-	ld8.fill r12=[r2],-16		// r2=&tf_r[FRAME_R10]
-(p1)	add	r1=-16,r1		// r1=&tf_r[FRAME_R11]
-	;;
-}
-{	.mmb
-	ld8.fill r11=[r1],-16		// r1=&tf_r[FRAME_R9]
-	ld8.fill r10=[r2],-16		// r2=&tf_r[FRAME_R8]
-	nop	22
-	;;
-}
-{	.mmi
-	ld8.fill r9=[r1],-16		// r1=&tf_r[FRAME_R7]
-	ld8.fill r8=[r2],-16		// r2=&tf_r[FRAME_R6]
-	nop	23
-	;;
-}
-{	.mmb
-	ld8.fill r7=[r1],-16		// r1=&tf_r[FRAME_R5]
-	ld8.fill r6=[r2],-16		// r2=&tf_r[FRAME_R4]
-	nop	24
-	;;
-}
-{	.mmi
-	ld8.fill r5=[r1],-16		// r1=&tf_r[FRAME_R3]
-	ld8.fill r4=[r2],-16		// r2=&tf_r[FRAME_R2]
-	nop	25
-	;;
-}
-{	.mmb
-	ld8.fill r3=[r1],-16		// r1=&tf_r[FRAME_R1]
-	ld8.fill r23=[r2],-16		// r2=&tf_b[7]
-	nop	26
-	;;
-}
-{	.mmi
-	ld8.fill r24=[r1],-16		// r1=&tf_b[6]
-	ld8	r16=[r2],-16		// r16=b7, r2=&tf_b[5]
-	nop	27
-	;;
-}
-{	.mmi
-	ld8	r17=[r1],-16		// r17=b6, r1=&tf_b[4]
-	ld8	r18=[r2],-16		// r18=b5, r2=&tf_b[3]
-	mov	b7=r16
-	;;
-}
-{	.mmi
-	ld8	r16=[r1],-16		// r16=b4, r1=&tf_b[2]
-	ld8	r19=[r2],-16		// r19=b3, r2=&tf_b[1]
-	mov	b6=r17
-	;;
-}
-{	.mii
-	ld8	r17=[r1],-16		// r17=b2, r1=&tf_b[0]
-	mov	b5=r18
-	mov	b4=r16
-	;;
-}
-{	.mii
-	ld8	r16=[r2],-16		// r16=b1, r2=&tf_ar_ec
-	mov	b3=r19
-	mov	b2=r17
-	;;
-}
-{	.mmi
-	ld8	r17=[r1],-16		// r17=b0, r1=&tf_ar_lc
-	ld8	r18=[r2],-16		// r18=ar.ec, r2=&tf_ar_fptr
-	mov	b1=r16
-	;;
-}
-{	.mmi
-	ld8	r16=[r1],-16		// r16=ar.lc, r1=&tf_ar_ccv
-	ld8	r19=[r2],-16		// r19=ar.fpsr, r1=&tf_ar_unat
-	mov	b0=r17
-	;;
-}
-{	.mmi
-	ld8	r17=[r1],-16		// r17=ar.ccv, r1=&tf_ndirty
-	mov	ar.fpsr=r19
-	mov	ar.ec=r18
-	;;
-}
-{	.mmi
-	ld8	r18=[r2],-16		// r18=ar.unat, r2=&tf_ar_rnat
-	mov	ar.ccv=r17
-	mov	ar.lc=r16
-	;;
-}
-{	.mmb
-	ld8	r27=[r1],-16	// r1=&tf_ar_bspstore
-	ld8	r21=[r2],-16		// r2=&tf_cr_ifs
-	nop	28
-	;;
-}
-{	.mmi
-	mov	ar.unat=r18
-	ld8	r22=[r1],-16	// r1=&tf_ar_pfs
-	nop	29
-}
-{	.mfb
-	ld8	r25=[r2],-16		// r2=&tf_ar_rsc
-	nop	30
-(p1)	br.cond.dpnt.few 1f		// don't switch bs if kernel
-	;;
-}
-{	.mmi
-	alloc	r16=ar.pfs,0,0,0,0	// discard current frame
-	;;
-	nop	31
-	shl	r16=r27,16		// value for ar.rsc
-	;;
-}
-{	.mmi
-	mov	ar.rsc=r16		// setup for loadrs
-	;;
-	loadrs				// load user regs from kernel bs
-	nop	32
-	;;
-}
-{	.mmi
-	mov	ar.bspstore=r22
-	;;
-	mov	ar.rnat=r21
-	nop	33
-	;;
-}
-1:
-{	.mmb
-	ld8	r19=[r1],-16		// r1=&tf_pr
-	ld8	r20=[r2],-16		// r2=&tf_cr_ifa
-	nop	34
-	;;
-}
-{	.mmi
-	ld8	r27=[r1],-16		// r1=&tf_cr_isr
-	ld8	r28=[r2],-16		// r2=&tf_cr_ipsr
-	mov	ar.pfs=r19
-	;;
-}
-{	.mmi
-	ld8	r29=[r1],-16		// r1=&tf_cr_iip
-	ld8	r30=[r2]
-	mov	pr=r27,0x1ffff
-	;;
-}
-{	.mmi
-	ld8	r31=[r1]
-	mov	cr.ifs=r25
-	mov	r2=r23
-	;;
-}
-{	.mmi
-	mov	cr.ifa=r28
-	mov	cr.iip=r31
-	mov	r1=r24
-	;;
-}
-{	.mmi
-	mov	cr.ipsr=r30
-	mov	ar.rsc=r20
-	nop	35
-	;;
-}
-{	.bbb
-	nop	36
-	nop	37
-	rfi
-	;;
-}
-END(exception_restore)
-	
-
-/*
- * exception_save: save interrupted state
- *
- * Arguments:
- *	b6	return address
- *	r16	saved b6
- *
- * Return:
- *	r14	cr.iim value for break traps
- *	sp	kernel stack pointer
- *	p1	true if user mode
- *	p2	true if kernel mode
- *	p3	true if interrupts were enabled
- */
-ENTRY(exception_save, 0)
-{	.mmi
-	rsm	psr.dt			// turn off data translations
-	;;
-	srlz.d				// serialize
-	mov	r27=pr
-}
-{	.mmi
-	mov	r30=cr.ipsr
-	;;
-	mov	r31=cr.iip
-	tbit.nz	p3,p0=r30,14		// check for interrupt enable state
-}
-{	.mmi
-	mov	r29=cr.isr
-	;; 
-	mov	r26=sp			// save sp
-	extr.u	r17=r30,32,2		// extract ipsr.cpl
-	;;
-}
-{	.mmi
-	cmp.eq	p1,p2=r0,r17		// test for kernel mode
-	;;
-(p2)	mov	sp=ar.k6		// and switch to kernel stack
-	mov	r24=r1
-	;;
-}
-{	.mii
-	mov	r28=cr.ifa
-	add	sp=-SIZEOF_TRAPFRAME,sp	// reserve trapframe
-	;;
-	dep	r1=0,sp,61,3		// r1=&tf_flags
-	;;
-}
-{	.mmi
-	st8	[r1]=r0,8		// zero flags, r1=&tf_cr_iip
-	;;
-	mov	r23=r2
-	add	r2=8,r1			// r2=&tf_cr_ipsr
-	;;
-}
-{	.mmb
-	st8	[r1]=r31,16		// r1=&tf_cr_isr
-	st8	[r2]=r30,16		// r2=&tf_cr_ifa
-	nop	1
-	;;
-}
-{	.mmb
-	st8	[r1]=r29,16		// r1=&tf_pr
-	st8	[r2]=r28,16		// r2=&tf_ar_rsc
-	nop	2
-	;;
-}
-{	.mmi
-	st8	[r1]=r27,16		// r1=&tf_cr_pfs
-	mov	r20=ar.rsc
-	mov	r19=ar.pfs
-	;;
-}
-{	.mmb
-	st8	[r2]=r20,16		// r2=&tf_cr_ifs
-	st8	[r1]=r19,16		// r1=&tf_ar_bspstore
-	cover
-	;;
-}
-{	.mmi
-	mov	ar.rsc=0
-	;;
-	mov	r22=ar.bspstore
-	;;
-}
-{	.mmi
-	mov	r25=cr.ifs
-	mov	r21=ar.rnat
-(p1)	mov	r31=r22		// so we can figure out ndirty
-	;;
-}
-{	.mmb
-(p2)	mov	r31=ar.k5		// kernel backing store
-	st8	[r2]=r25,16		// r2=&tf_ar_rnat
-	nop	3
-	;;
-}
-{	.mmi
-	st8	[r1]=r22,16	// r1=&tf_ndirty
-(p2)	mov	ar.bspstore=r31		// switch bspstore
-	nop	4
-	;;
-}
-{	.mmi
-	mov	r17=ar.bsp
-	;;
-	st8	[r2]=r21,16		// r2=&tf_ar_unat
-	sub	r17=r17,r31		// ndirty (in bytes)
-	;;
-}
-{	.mmi
-	st8	[r1]=r17,16		// r1=&tf_ar_ccv
-	mov	ar.rsc=3		// switch RSE back on
-	mov	r18=ar.lc
-}
-{	.mmi
-	mov	r31=ar.unat
-	;;
-	st8	[r2]=r31,16		// r2=&tf_ar_fpsr
-	mov	r19=ar.ec
-}
-{	.mmi
-	mov	r17=ar.ccv
-	;;
-	st8	[r1]=r17,16		// r1=&tf_ar_lc
-	nop	5
-	;;
-}
-{	.mmi
-	mov	r31=ar.fpsr
-	;;
-	st8	[r2]=r31,16		// r2=&tf_ar_ec
-	mov	r30=b0
-	;;
-}
-{	.mmi
-	st8	[r1]=r18,16		// r1=&tf_b[0]
-	;;
-	st8	[r2]=r19,16		// r2=&tf_b[1]
-	mov	r31=b1
-}
-{	.mmi
-	st8	[r1]=r30,16		// r1=&tf_b[2]
-	;;
-	st8	[r2]=r31,16		// r2=&tf_b[3]
-	mov	r31=b2
-	;;
-}
-{	.mii
-	st8	[r1]=r31,16		// r1=&tf_b[4]
-	mov	r17=b3
-	;;
-	mov	r18=b4
-}
-{	.mmi
-	st8	[r2]=r17,16		// r2=&tf_b[5]
-	;; 
-	st8	[r1]=r18,16		// r1=&tf_b[6]
-	mov	r31=b5
-	;;
-}
-{	.mii
-	st8	[r2]=r31,16		// r2=&tf_b[7]
-	mov	r18=b7
-}
-{	.mmi
-	st8	[r1]=r16,16		// r1=&tf_r[FRAME_R1]
-	;; 
-	st8	[r2]=r18,16		// r2=&tf_r[FRAME_R2]
-	nop	7
-	;;
-}
-{	.mmb
-	.mem.offset 0,0
-	st8.spill [r1]=r24,16		// r1=&tf_r[FRAME_R3]
-	.mem.offset 8,0
-	st8.spill [r2]=r23,16		// r2=&tf_r[FRAME_R4]
-	nop	8
-	;;
-}
-{	.mmi
-	.mem.offset 16,0
-	st8.spill [r1]=r3,16		// r1=&tf_r[FRAME_R5]
-	.mem.offset 32,0
-	st8.spill [r2]=r4,16		// r2=&tf_r[FRAME_R6]
-	nop	9
-	;;
-}
-{	.mmb
-	.mem.offset 48,0
-	st8.spill [r1]=r5,16		// r1=&tf_r[FRAME_R7]
-	.mem.offset 64,0
-	st8.spill [r2]=r6,16		// r2=&tf_r[FRAME_R8]
-	nop	10
-	;;
-}
-{	.mmi
-	.mem.offset 80,0
-	st8.spill [r1]=r7,16		// r1=&tf_r[FRAME_R9]
-	.mem.offset 96,0
-	st8.spill [r2]=r8,16		// r2=&tf_r[FRAME_R10]
-	nop	11
-	;;
-}
-{	.mmb
-	.mem.offset 112,0
-	st8.spill [r1]=r9,16		// r1=&tf_r[FRAME_R11]
-	.mem.offset 128,0
-	st8.spill [r2]=r10,16		// r2=&tf_r[FRAME_SP]
-	nop	12
-	;;
-}
-{	.mmi
-	.mem.offset 144,0
-	st8.spill [r1]=r11,16		// r1=&tf_r[FRAME_R13]
-	.mem.offset 160,0
-	st8.spill [r2]=r26,16		// r2=&tf_r[FRAME_R14]
-	nop	13
-	;;
-}
-{	.mmb
-	.mem.offset 176,0
-	st8.spill [r1]=r13,16		// r1=&tf_r[FRAME_R15]
-	.mem.offset 192,0
-	st8.spill [r2]=r14,16		// r2=&tf_r[FRAME_R16]
-	nop	14
-	;;
-}
-{	.mfb
-	.mem.offset 208,0
-	st8.spill [r1]=r15,16		// r1=&tf_r[FRAME_R17]
-	nop	15
-	bsw.1				// switch to bank 1
-	;;
-}
-{	.mmi
-	.mem.offset 224,0
-	st8.spill [r2]=r16,16		// r2=&tf_r[FRAME_R18]
-	.mem.offset 240,0
-	st8.spill [r1]=r17,16		// r1=&tf_r[FRAME_R19]
-	nop	16
-	;;
-}
-{	.mmb
-	.mem.offset 256,0
-	st8.spill [r2]=r18,16		// r2=&tf_r[FRAME_R20]
-	.mem.offset 272,0
-	st8.spill [r1]=r19,16		// r1=&tf_r[FRAME_R21]
-	nop	17
-	;;
-}
-{	.mmi
-	.mem.offset 288,0
-	st8.spill [r2]=r20,16		// r2=&tf_r[FRAME_R22]
-	.mem.offset 304,0
-	st8.spill [r1]=r21,16		// r1=&tf_r[FRAME_R23]
-	nop	18
-	;;
-}
-{	.mmb
-	.mem.offset 320,0
-	st8.spill [r2]=r22,16		// r2=&tf_r[FRAME_R24]
-	.mem.offset 336,0
-	st8.spill [r1]=r23,16		// r1=&tf_r[FRAME_R25]
-	nop	19
-	;;
-}
-{	.mmi
-	.mem.offset 352,0
-	st8.spill [r2]=r24,16		// r2=&tf_r[FRAME_R26]
-	.mem.offset 368,0
-	st8.spill [r1]=r25,16		// r1=&tf_r[FRAME_R27]
-	nop	20
-	;;
-}
-{	.mmb
-	.mem.offset 384,0
-	st8.spill [r2]=r26,16		// r2=&tf_r[FRAME_R28]
-	.mem.offset 400,0
-	st8.spill [r1]=r27,16		// r1=&tf_r[FRAME_R29]
-	nop	21
-	;;
-}
-{	.mmi
-	.mem.offset 416,0
-	st8.spill [r2]=r28,16		// r2=&tf_r[FRAME_R30]
-	.mem.offset 432,0
-	st8.spill [r1]=r29,16		// r1=&tf_r[FRAME_R31]
-	nop	22
-	;;
-}
-{	.mmb
-	.mem.offset 448,0
-	st8.spill [r2]=r30,16		// r2=&tf_f[FRAME_F6]
-	.mem.offset 464,0
-	st8.spill [r1]=r31,24		// r1=&tf_f[FRAME_F7]
-	nop	23
-	;;
-}
-{	.mmi
-	stf.spill [r2]=f6,32		// r2=&tf_f[FRAME_F8]
-	stf.spill [r1]=f7,32		// r1=&tf_f[FRAME_F9]
-	nop	24
-	;;
-}
-{	.mmb
-	stf.spill [r2]=f8,32		// r2=&tf_f[FRAME_F10]
-	stf.spill [r1]=f9,32		// r1=&tf_f[FRAME_F11]
-	nop	25
-	;;
-}
-{	.mmi
-	stf.spill [r2]=f10,32		// r2=&tf_f[FRAME_F12]
-	stf.spill [r1]=f11,32		// r1=&tf_f[FRAME_F13]
-	nop	26
-	;;
-}
-{	.mmb
-	stf.spill [r2]=f12,32		// r2=&tf_f[FRAME_F14]
-	stf.spill [r1]=f13,32		// r1=&tf_f[FRAME_F15]
-	nop	27
-	;;
-}
-{	.mmi
-	stf.spill [r2]=f14		//
-	stf.spill [r1]=f15		//
-	nop	28
-	;;
-}
-{	.mlx
-	mov	r14=cr.iim		// break immediate
-	movl	r1=__gp			// kernel globals
-}
-{	.mmi
-	ssm	psr.ic|psr.dt		// enable interrupts & translation
-	;;
-	srlz.i				// serialize
-	nop	29
-	;;
-}
-{	.mfb
-	mov	r13=ar.k4		// processor globals
-	nop	30
-	br.sptk.few b6			// not br.ret - we were not br.call'ed
-	;;
-}
-END(exception_save)
-	
-/*
- * System call entry point (via Break Instruction vector).
- *
- * Arguments:
- *	r15		System call number
- *	out0-out7	System call arguments
- */
-ENTRY(do_syscall, 0)
-	.prologue
-	.unwabi	@svr4, 'I'
-	.save	rp,r0
-	.body
-	// Save minimal state for syscall.
-	// We need to save enough state so that sendsig doesn't
-	// trash things if we take a signal during the system call.
-	// Essentially we need to save all the function-preserved
-	// state. Note that if we don't take a signal, we don't need
-	// to restore much of that state on the way out. Note also
-	// that when we save r4-r7 we spill their NaT bits into
-	// ar.unat. This register is preserved by the call to
-	// syscall() and if a full restore is needed,
-	// exception_restore will recover the NaT bits from ar.unat.
-	// The function-preserved state (including syscall number) is:
-	//
-	//	r1,r4-r7,sp,r15
-	//	f16-f31
-	//	p1-p5,p16-p63
-	//	b0-b5
-	//	various ar's
-	//
-{ .mmi					// start reading high latency regs
-	mov	r16=cr.ipsr		// (13)
-	mov.m	r17=ar.rsc		// (13)
-	mov	r18=sp			// save user sp
-	;;
-} { .mmi
-	mov	sp=ar.k6		// (13) kernel sp
-	mov	r19=cr.isr		// (13)
-	nop.i	0
-} { .mmi
-	mov.m	ar.rsc=0
-	;; 
-	mov.m	r20=ar.bspstore		// (13)
-	nop.i	0
-} { .mmi
-	mov.m	r21=ar.k5		// (13)
-	mov.m	r22=ar.rnat		// (6)
-	nop.i	0
-} { .mmi
-	mov.m	r23=ar.unat		// (6)
-	rsm	psr.dt			// (5) physical addressing
-} { .mii
-	mov	r24=cr.iip		// (2)
-	mov.i	r25=ar.pfs		// (2)
-	add	sp=-SIZEOF_TRAPFRAME,sp // reserve trapframe
-	;; 
-} { .mii
-	addl	r27=FRAME_SYSCALL,r0	// (1)
-	mov	r26=pr			// (2)
-	dep	r30=0,sp,61,3		// physical address
-} { .mmi
-	srlz.d				// serialize psr.dt
-	;; 
-	add	r31=8,r30		// secondary pointer
-	;; 
-} { .mmi
-	st8	[r30]=r27,16		// tf_flags
-	st8	[r31]=r24,16		// save cr.iip
-	mov	r28=b0
-	;;
-} { .mmi
-	st8	[r30]=r16,24		// save cr.ipsr, skip to pr
-	st8	[r31]=r19,24		// save cr.isr, skip to ar.rsc
-	mov	r24=b1
-	;;
-} { .mmi
-	st8	[r30]=r26,16		// save pr, skip to ar.pfs
-	st8	[r31]=r17,24		// save ar.rsc, skip to ar.bspstore
-	mov	r27=b2
-	;;
-} { .mii
-	st8	[r30]=r25,24		// save ar.pfs, skip to ar.rnat
-	mov	r16=b3
-	mov	r17=b4
-	;;
-} { .mmi
-	st8	[r31]=r20,24		// save ar.bspstore, skip to ar.unat
-	mov.m	ar.bspstore=r21		// switch to kernel backing store
-	mov	r29=b5
-	;;
-} { .mmi
-	mov.m	r20=ar.ccv
-	mov.m	r21=ar.fpsr
-	nop.i	0
-	;; 
-} { .mmi
-	st8	[r30]=r22,24		// save ar.rnat, skip to ar.ccv
-	st8	[r31]=r23,16		// save ar.unat, skip to ar.fpsr
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r20,32		// save ar.ccv, skip to b0
-	st8	[r31]=r21,32		// save ar.fpsr, skip to b1
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r28,16		// save b0, skip to b2
-	st8	[r31]=r24,16		// save b1, skip to b3
-	nop.i	0
-	;; 
-} { .mmi
-	st8	[r30]=r27,16		// save b2, skip to b4
-	st8	[r31]=r16,16		// save b3, skip to b5
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r17,TF_R_R1-(TF_B+4*8) // save b4, skip to r1
-	st8	[r31]=r29,TF_R_R4-(TF_B+5*8) // save b5, skip to r4
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r1,TF_R_R5-TF_R_R1 // save r1, skip to r5
-	.mem.offset 8,0
-	st8.spill [r31]=r4,16		// save r4, skip to r6
-	nop.i	0
-	;;
-} { .mmi
-	.mem.offset 0,0
-	st8.spill [r30]=r5,16		// save r5, skip to r7
-	.mem.offset 8,0
-	st8.spill [r31]=r6,TF_R_SP-TF_R_R6 // save r6, skip to sp
-	nop.i	0
-	;;
-} { .mmi
-	.mem.offset 0,0
-	st8.spill [r30]=r7,TF_R_R15-TF_R_R7 // save r7, skip to r15
-	st8	[r31]=r18		// save sp
-	nop.i	0
-	;; 
-} { .mmb
-	st8	[r30]=r15		// save r15 (syscall number)
-	add	sp=-(8*8),sp		// reserve stack for arguments
-	br.call.sptk.few b0=Lsaveargs	// dump args
-} { .mmb
-	mov.m	r13=ar.k4		// processor globals
-	nop.m	0
-	bsw.1				// switch back to bank 1
-	;;
-} { .mmb
-	mov	r16=sp			// point at args
-	mov.m	r17=ar.k5		// for calculating ndirty
-	cover				// preserve user register frame
-	;;
-} { .mmi
-	mov	r18=cr.ifs		// record user's CFM
-	mov.m	r19=ar.bsp		// ndirty = ar.bsp - kbsp
-	add	sp=-16,sp		// reserve scratch space
-	;;
-} { .mmi
-	add	r20=TF_CR_IFS+(8*8),r16	// point at cr.ifs
-	ssm	psr.ic|psr.dt		// reenable traps and translation
-	sub	r19=r19,r17		// calculate ndirty
-	;;
-} { .mmi
-	srlz.i				// serialize psr.ic and psr.dt
-	;;
-	ssm	psr.i			// safe to take interrupts again
-	add	r21=TF_NDIRTY+(8*8),r16	// point at ndirty
-	;;
-} { .mmi
-	st8	[r20]=r18		// save cr.ifs
-	st8	[r21]=r19		// save ndirty
-	;;
-} { .mmi
-	alloc	r14=ar.pfs,0,1,3,0
-	srlz.d				// serialize psr.i
-	add	loc0=(8*8),r16		// remember where trapframe is
-	;;
-} { .mlx
-	mov	out0=r15		// syscall number (from user)
-	movl	gp=__gp			// kernel globals
-} { .mmb
-	mov	out1=r16		// arguments
-	add	out2=(8*8),r16		// trapframe pointer
-	br.call.sptk.many rp=syscall	// do the work
-} { .mmi
-3:	rsm	psr.i			// we know that psr.i == 1
-	add	r14=PC_CURTHREAD,r13	// &curthread
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r14=[r14]		// curthread
-	;;
-	nop.i	0
-	;;
-} { .mmi
-	;;
-	add	r14=TD_FLAGS,r14	// &curthread->td_flags
-	nop.i	0
-	;;
-} { .mmi
-	ld4	r14=[r14]		// curkse->ke_flags
-	;;
-	nop.m	0
-	tbit.nz	p6,p7=r14,11		// TDF_ASTPENDING
-	;;
-} { .mib
-	nop.m	0
-(p7)	tbit.nz.or.andcm p6,p7=r14,16	// TDF_NEEDRESCHED
-(p7)	br.cond.dptk 2f
-	;;
-} { .mmi
-	ssm	psr.i			// restore interrupts
-	;;
-	srlz.d
-	mov	out0=loc0		// trapframe argument to ast()
-} { .mib
-	nop.m	0
-	nop.i	0
-	br.call.sptk.many rp=ast
-} { .mib
-	nop.m	0
-	nop.i	0
-	br	3b
-} { .mii
-2:	ld8	r14=[loc0]		// check tf_flags
-	dep	r15=0,loc0,61,3		// physical address of trapframe
-	;;
-	tbit.z p6,p0=r14,0		// check FRAME_SYSCALL bit
-	;;
-} { .mib
-(p6)	add	sp=-16,loc0		// do a full restore if clear
-	add	r16=SIZEOF_TRAPFRAME,loc0 // new kernel sp
-(p6)	br.dpnt.many exception_restore
-} { .mmi
-	rsm 	psr.dt|psr.ic|psr.i	// get ready to restore
-	;;
-	srlz.i				// serialise psr.dt and psr.ic
-	add	r30=TF_R_R15,r15	// point at r15
-	;; 
-} { .mmi
-	alloc	r14=ar.pfs,0,0,0,0	// discard register frame
-	mov	ar.k6=r16		// restore kernel sp
-	add	r31=TF_R_SP,r15		// point at sp
-	;;
-} { .mmi
-	ld8	r15=[r30],TF_R_R10-TF_R_R15 // restore r15, skip to r10
-	ld8	sp=[r31],TF_R_R9-TF_R_SP // restore user sp, skip to r9
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r10=[r30],-16		// restore r10, skip to r8
-	ld8	r9=[r31],TF_R_R1-TF_R_R9 // restore r9, skip to r1
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r8=[r30],TF_B-TF_R_R8	// restore r8, skip to b0
-	ld8	r1=[r31],TF_AR_FPSR-TF_R_R1 // restore r1, skip to ar.fpsr
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r16=[r30],-32		// restore b0, skip to ar.ccv
-	ld8	r17=[r31],-16		// restore ar.fpsr, skip to ar.unat
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r18=[r30],-16		// restore ar.ccv, skip to ndirty
-	ld8	r19=[r31],-16		// restore ar.unat, skip to ar.rnat
-	mov	b0=r16
-	;;
-} { .mmi
-	ld8	r20=[r30],-16		// restore ndirty, skip to ar.bspstore
-	ld8	r21=[r31],-16		// restore ar.rnat, skip to cr.ifs
-	nop.i	0
-	;; 
-} { .mmi
-	ld8	r16=[r30],-16		// restore ar.bspstore, skip to ar.pfs
-	mov	ar.fpsr=r17
-	shl	r20=r20,16		// value for ar.rsc
-	;; 
-} { .mmi
-	ld8	r22=[r31],-16		// restore cr.ifs, skip to ar.rsc
-	mov.m	ar.ccv=r18
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r17=[r30],-16		// restore ar.pfs, skip to pr
-	mov.m	ar.unat=r19
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r18=[r31],-32		// restore ar.rsc, skip to cr.ipsr
-	mov.m	ar.rsc=r20		// setup for loadrs
-	nop.i	0
-	;;
-} { .mmi
-	loadrs				// restore user stacked registers
-	;; 
-	mov.m	ar.bspstore=r16		// back to user backing store
-	mov.i	ar.pfs=r17	
-	;;
-} { .mmi
-	mov.m	ar.rnat=r21
-	mov.m	ar.rsc=r18
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r16=[r30],-32		// restore pr, skip to cr.iip
-	ld8	r17=[r31]		// restore cr.ipsr
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r18=[r30]		// restore cr.iip
-	mov	cr.ifs=r22
-	nop.i	0
-	;;
-} { .mmi
-	mov	cr.iip=r18
-	mov	cr.ipsr=r17
-	mov	pr=r16,0x1ffff
-	;;
-} { .bbb
-	rfi
-}	
-
-	// This is done as a function call to make sure that we only
-	// have output registers in the register frame. It also gives
-	// us a chance to use alloc to round up to 8 arguments for
-	// simplicity.
-	//
-	// We are still running in physical mode with psr.ic==0 because
-	// we haven't yet covered the user's register frame to get a
-	// value for cr.ifs
-Lsaveargs:
-{ .mii
-	alloc	r14=ar.pfs,0,0,8,0	// round up to 8 outputs
-	extr.u	r31=sp,0,61		// physical address
-	;;
-	add	r30=8,r31
-	;;
-} { .mmi
-	st8	[r31]=r32,16
-	st8	[r30]=r33,16
-	;;
-} { .mmi
-	st8	[r31]=r34,16
-	st8	[r30]=r35,16
-	;; 
-} { .mmi
-	st8	[r31]=r36,16
-	st8	[r30]=r37,16
-	;; 
-} { .mmb
-	st8	[r31]=r38
-	st8	[r30]=r39
-	br.ret.sptk.many b0
-}
-	.global do_syscall_end
-do_syscall_end:
-
-END(do_syscall)
diff --git a/sys/ia64/ia64/exception.s b/sys/ia64/ia64/exception.s
index 9e89aaa..947df2d 100644
--- a/sys/ia64/ia64/exception.s
+++ b/sys/ia64/ia64/exception.s
@@ -1,4 +1,5 @@
 /*-
+ * Copyright (c) 2003 Marcel Moolenaar
  * Copyright (c) 2000 Doug Rabson
  * All rights reserved.
  *
@@ -31,12 +32,541 @@
 #include <assym.s>
 
 /*
- * ar.k7 = curthread
- * ar.k6 = ksp
- * ar.k5 = kbsp
- * ar.k4 = pcpup
+ * ar.k7 = kernel memory stack
+ * ar.k6 = kernel register stack
+ * ar.k5 = EPC gateway page
+ * ar.k4 = PCPU data
  */
 
+	.text
+
+/*
+ * exception_save: save interrupted state
+ *
+ * Arguments:
+ *	r16	address of bundle that contains the branch. The
+ *		return address will be the next bundle.
+ *	r17	the value to save as ifa in the trapframe. This
+ *		normally is cr.ifa, but some interruptions set
+ *		set cr.iim and not cr.ifa.
+ *
+ * Returns:
+ *	p15	interrupted from user stack
+ *	p14	interrupted from kernel stack
+ *	p13	interrupted from user backing store
+ *	p12	interrupted from kernel backing store
+ *	p11	interrupts were enabled
+ *	p10	interrupts were disabled
+ */
+ENTRY(exception_save, 0)
+{	.mii
+	mov		r20=ar.unat
+	extr.u		r31=sp,61,3
+	mov		r18=pr
+	;;
+}
+{	.mmi
+	cmp.le		p14,p15=5,r31
+	;;
+(p15)	mov		r23=ar.k7		// kernel memory stack
+(p14)	mov		r23=sp
+	;;
+}
+{	.mii
+	mov		r21=ar.rsc
+	add		r30=-SIZEOF_TRAPFRAME,r23
+	;;
+	dep		r30=0,r30,0,10
+	;;
+}
+{	.mmi
+	mov		ar.rsc=0
+	sub		r19=r23,r30
+	add		r31=8,r30
+	;;
+}
+{	.mlx
+	mov		r22=cr.iip
+	movl		r26=exception_save_restart
+	;;
+}
+
+	/*
+	 * We have a 1KB aligned trapframe, pointed to by sp. If we write
+	 * to the trapframe, we may trigger a data nested TLB fault. By
+	 * aligning the trapframe on a 1KB boundary, we guarantee that if
+	 * we get a data nested TLB fault, it will be on the very first
+	 * write. Since the data nested TLB fault does not preserve any
+	 * state, we have to be careful what we clobber. Consequently, we
+	 * have to be careful what we use here. Below a list of registers
+	 * that are currently alive:
+	 *	r16,r17=arguments
+	 *	r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS
+	 *	r26=restart point
+	 *	r30,r31=trapframe pointers
+	 *	p14,p15=memory stack switch
+	 */
+exception_save_restart:
+{	.mmi
+	st8		[r30]=r19,16		// length
+	st8		[r31]=r0,16		// flags
+	add		r19=16,r19
+	;;
+}
+{	.mmi
+	st8.spill	[r30]=sp,16		// sp
+	st8		[r31]=r20,16		// unat
+	sub		sp=r23,r19
+	;;
+}
+{	.mmi
+	mov		r19=ar.rnat
+	mov		r20=ar.bspstore
+	mov		r23=rp
+	;;
+}
+	// r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp
+{	.mmi
+	st8		[r30]=r23,16		// rp
+	st8		[r31]=r18,16		// pr
+	mov		r24=ar.pfs
+	;;
+}
+{	.mmb
+	st8		[r30]=r24,16		// pfs
+	st8		[r31]=r20,16		// bspstore
+	cover
+	;;
+}
+{	.mmi
+	mov		r18=ar.fpsr
+	mov		r23=cr.ipsr
+	extr.u		r24=r20,61,3
+	;;
+}
+	// r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr
+{	.mmi
+	st8		[r30]=r19,16		// rnat
+	st8		[r31]=r0,16		// __spare
+	cmp.le		p12,p13=5,r24
+	;;
+}
+{	.mmi
+	st8.spill	[r30]=r13,16		// tp
+	st8		[r31]=r21,16		// rsc
+	tbit.nz		p11,p10=r23,14		// p11=interrupts enabled
+	;;
+}
+{	.mmi
+	st8		[r30]=r18,16		// fpsr
+(p13)	mov		r20=ar.k6		// kernel register stack
+	nop		0
+	;;
+}
+	// r20=bspstore, r22=iip, r23=ipsr
+{	.mmi
+	st8		[r31]=r23,16		// psr
+	mov		ar.bspstore=r20
+	nop		0
+	;;
+}
+{	.mmi
+	mov		r18=ar.bsp
+	;;
+	mov		r19=cr.ifs
+	sub		r18=r18,r20
+	;;
+}
+{	.mmi
+	st8.spill	[r30]=gp,16		// gp
+	st8		[r31]=r18,16		// ndirty
+	nop		0
+	;;
+}
+	// r19=ifs, r22=iip
+{	.mmi
+	st8		[r30]=r19,16		// cfm
+	st8		[r31]=r22,16		// iip
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r30]=r17		// ifa
+	mov		r18=cr.isr
+	add		r29=16,r30
+	;;
+}
+{	.mmi
+	st8		[r31]=r18		// isr
+	add		r30=8,r29
+	add		r31=16,r29
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r2,16		// r2
+	.mem.offset	8,0
+	st8.spill	[r31]=r3,16		// r3
+	add		r2=9*8,r29
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r8,16		// r8
+	.mem.offset	8,0
+	st8.spill	[r31]=r9,16		// r9
+	add		r3=8,r2
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r10,16		// r10
+	.mem.offset	8,0
+	st8.spill	[r31]=r11,16		// r11
+	add		r8=16,r16
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r30]=r14		// r14
+	.mem.offset	8,0
+	st8.spill	[r31]=r15		// r15
+	mov		r9=r29
+}
+{	.mmb
+	mov		r10=ar.csd
+	mov		r11=ar.ssd
+	bsw.1
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r16,16		// r16
+	.mem.offset	8,0
+	st8.spill	[r3]=r17,16		// r17
+	mov		r14=b6
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r18,16		// r18
+	.mem.offset	8,0
+	st8.spill	[r3]=r19,16		// r19
+	mov		r15=b7
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r20,16		// r20
+	.mem.offset	8,0
+	st8.spill	[r3]=r21,16		// r21
+	mov		b7=r8
+	;;
+}
+{	.mmi
+	.mem.offset	0,0
+	st8.spill	[r2]=r22,16		// r22
+	.mem.offset	8,0
+	st8.spill	[r3]=r23,16		// r23
+	;;
+}
+
+	.mem.offset	0,0
+	st8.spill	[r2]=r24,16		// r24
+	.mem.offset	8,0
+	st8.spill	[r3]=r25,16		// r25
+	;;
+	.mem.offset	0,0
+	st8.spill	[r2]=r26,16		// r26
+	.mem.offset	8,0
+	st8.spill	[r3]=r27,16		// r27
+	;;
+	.mem.offset	0,0
+	st8.spill	[r2]=r28,16		// r28
+	.mem.offset	8,0
+	st8.spill	[r3]=r29,16		// r29
+	;;
+	.mem.offset	0,0
+	st8.spill	[r2]=r30,16		// r30
+	.mem.offset	8,0
+	st8.spill	[r3]=r31,16		// r31
+	;;
+
+{	.mmi
+	st8		[r2]=r14,16		// b6
+	mov		r17=ar.unat
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r3]=r15,16		// b7
+	mov		r16=ar.ccv
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r2]=r16,16		// ccv
+	st8		[r3]=r10,16		// csd
+	nop		0
+	;;
+}
+{	.mmi
+	st8		[r2]=r11,24		// ssd
+	st8		[r9]=r17
+	nop		0
+	;;
+}
+
+	stf.spill	[r3]=f6,32		// f6
+	stf.spill	[r2]=f7,32		// f7
+	;;
+	stf.spill	[r3]=f8,32		// f8
+	stf.spill	[r2]=f9,32		// f9
+	;;
+	stf.spill	[r3]=f10,32		// f10
+	stf.spill	[r2]=f11,32		// f11
+	;;
+	stf.spill	[r3]=f12,32		// f12
+	stf.spill	[r2]=f13,32		// f13
+	;;
+	stf.spill	[r3]=f14		// f14
+	stf.spill	[r2]=f15		// f15
+	;;
+{	.mmi
+	mov		ar.rsc=3
+	mov		r13=ar.k4
+	nop		0
+	;;
+}
+{	.mlx
+	ssm		psr.ic|psr.dfh
+	movl		gp=__gp
+	;;
+}
+{	.mfb
+	srlz.d
+	nop		0
+	br.sptk		b7
+	;;
+}
+END(exception_save)
+
+/*
+ * exception_restore:	restore interrupted state
+ *
+ * Arguments:
+ *	sp+16	trapframe pointer
+ */
+ENTRY(exception_restore, 0)
+{	.mmi
+	rsm		psr.i
+	add		sp=16,sp
+	nop		0
+	;;
+}
+{	.mmi
+	add		r3=SIZEOF_TRAPFRAME-32,sp
+	add		r2=SIZEOF_TRAPFRAME-16,sp
+	add		r8=SIZEOF_SPECIAL+16,sp
+	;;
+}
+	// The next load can trap. Let it be...
+	ldf.fill	f15=[r2],-32		// f15
+	ldf.fill	f14=[r3],-32		// f14
+	;;
+	ldf.fill	f13=[r2],-32		// f13
+	ldf.fill	f12=[r3],-32		// f12
+	;;
+	ldf.fill	f11=[r2],-32		// f11
+	ldf.fill	f10=[r3],-32		// f10
+	;;
+	ldf.fill	f9=[r2],-32		// f9
+	ldf.fill	f8=[r3],-32		// f8
+	;;
+	ldf.fill	f7=[r2],-24		// f7
+	ldf.fill	f6=[r3],-16		// f6
+	;;
+
+{	.mmi
+	ld8		r8=[r8]			// unat (after)
+	;;
+	mov		ar.unat=r8
+	nop		0
+	;;
+}
+
+	ld8		r10=[r2],-16		// ssd
+	ld8		r11=[r3],-16		// csd
+	;;
+	mov		ar.ssd=r10
+	mov		ar.csd=r11
+
+	ld8		r14=[r2],-16		// ccv
+	ld8		r15=[r3],-16		// b7
+	;;
+
+{	.mmi
+	mov		ar.ccv=r14
+	ld8		r8=[r2],-16		// b6
+	mov		b7=r15
+	;;
+}
+{	.mmi
+	ld8.fill	r31=[r3],-16		// r31
+	ld8.fill	r30=[r2],-16		// r30
+	mov		b6=r8
+	;;
+}
+
+	ld8.fill	r29=[r3],-16		// r29
+	ld8.fill	r28=[r2],-16		// r28
+	;;
+	ld8.fill	r27=[r3],-16		// r27
+	ld8.fill	r26=[r2],-16		// r26
+	;;
+	ld8.fill	r25=[r3],-16		// r25
+	ld8.fill	r24=[r2],-16		// r24
+	;;
+	ld8.fill	r23=[r3],-16		// r23
+	ld8.fill	r22=[r2],-16		// r22
+	;;
+	ld8.fill	r21=[r3],-16		// r21
+	ld8.fill	r20=[r2],-16		// r20
+	;;
+	ld8.fill	r19=[r3],-16		// r19
+	ld8.fill	r18=[r2],-16		// r18
+	;;
+
+{	.mmb
+	ld8.fill	r17=[r3],-16		// r17
+	ld8.fill	r16=[r2],-16		// r16
+	bsw.0
+	;;
+}
+{	.mmi
+	ld8.fill	r15=[r3],-16		// r15
+	ld8.fill	r14=[r2],-16		// r14
+	add		r31=16,sp
+	;;
+}
+{	.mmi
+	ld8		r16=[sp]		// tf_length
+	ld8.fill	r11=[r3],-16		// r11
+	add		r30=24,sp
+	;;
+}
+{	.mmi
+	ld8.fill	r10=[r2],-16		// r10
+	ld8.fill	r9=[r3],-16		// r9
+	add		r16=r16,sp		// ar.k7
+	;;
+}
+{	.mmi
+	ld8.fill	r8=[r2],-16		// r8
+	ld8.fill	r3=[r3]			// r3
+	;;
+}
+	// We want nested TLB faults from here on...
+	rsm		psr.ic|psr.i
+	ld8.fill	r2=[r2]			// r2
+	nop		0
+	;;
+	srlz.d
+	ld8.fill	sp=[r31],16		// sp
+	nop		0
+	;;
+
+	ld8		r17=[r30],16		// unat
+	ld8		r29=[r31],16		// rp
+	;;
+	ld8		r18=[r30],16		// pr
+	ld8		r28=[r31],16		// pfs
+	mov		rp=r29
+	;;
+	ld8		r20=[r30],24		// bspstore
+	ld8		r21=[r31],24		// rnat
+	mov		ar.pfs=r28
+	;;
+	ld8.fill	r29=[r30],16		// tp
+	ld8		r22=[r31],16		// rsc
+	;;
+{	.mmi
+	ld8		r23=[r30],16		// fpsr
+	ld8		r24=[r31],16		// psr
+	extr.u		r28=r20,61,3
+	;;
+}
+{	.mmi
+	ld8.fill	r1=[r30],16		// gp
+	ld8		r25=[r31],16		// ndirty
+	cmp.le		p14,p15=5,r28
+	;;
+}
+{	.mmb
+	ld8		r26=[r30]		// cfm
+	ld8		r19=[r31]		// ip
+(p14)	br.cond.sptk	1f
+	;;
+}
+{	.mib
+	// Switch register stack
+	alloc		r30=ar.pfs,0,0,0,0	// discard current frame
+	shl		r31=r25,16		// value for ar.rsc
+	nop		0
+	;;
+}
+	// The loadrs can fault if the backing store is not currently
+	// mapped. We assured forward progress by getting everything we
+	// need from the trapframe so that we don't care if the CPU
+	// purges that translation when it needs to insert a new one for
+	// the backing store.
+{	.mmi
+	mov		ar.rsc=r31		// setup for loadrs
+	mov		ar.k7=r16
+	mov		r13=r29
+	;;
+}
+exception_restore_restart:
+{	.mmi
+	mov		r30=ar.bspstore
+	;;
+	loadrs					// load user regs
+	nop		0
+	;;
+}
+{	.mmi
+	mov		r31=ar.bspstore
+	;;
+	mov		ar.bspstore=r20
+	dep		r31=0,r31,0,9
+	;;
+}
+{	.mmb
+	mov		ar.k6=r31
+	mov		ar.rnat=r21
+	nop		0
+	;;
+}
+1:
+{	.mmb
+	mov		ar.unat=r17
+	mov		cr.iip=r19
+	nop		0
+}
+{	.mmi
+	mov		cr.ipsr=r24
+	mov		cr.ifs=r26
+	mov		pr=r18,0x1fffe
+	;;
+}
+{	.mmb
+	mov		ar.rsc=r22
+	mov		ar.fpsr=r23
+	rfi
+	;;
+}
+END(exception_restore)
+
 /*
  * Call exception_save_regs to preserve the interrupted state in a
  * trapframe. Note that we don't use a call instruction because we
@@ -46,20 +576,27 @@
  * which will restore the interrupted state before executing an rfi to
  * resume it.
  */
-#define TRAP(_n_)				\
-1:	mov	r17=ip;;			\
-	add	r17=2f-1b,r17;			\
-	mov	r16=b6;;			\
-	mov	b6=r17;				\
-	br.sptk.few exception_save;		\
-2: (p3)	ssm	psr.i;				\
-	alloc	r15=ar.pfs,0,0,3,0;		\
-	mov	out0=_n_;			\
-	mov	out1=r14;			\
-	mov	out2=sp;;			\
-	add	sp=-16,sp;;			\
-	br.call.sptk.few rp=trap;		\
-3:	br.sptk.many exception_restore
+#define TRAP(_n_, _ifa_)			\
+{	.mib ;					\
+	mov		r17=_ifa_ ;		\
+	mov		r16=ip ;		\
+	br.sptk		exception_save ;	\
+} ;						\
+{	.mmi ;					\
+(p11)	ssm		psr.i ;;		\
+	alloc		r15=ar.pfs,0,0,2,0 ;	\
+	mov		out0=_n_ ;;		\
+} ;						\
+{	.mfb ;					\
+	add		out1=16,sp ;		\
+	nop		0 ;			\
+	br.call.sptk	rp=trap ;		\
+} ;						\
+{	.mfb ;					\
+	nop		0 ;			\
+	nop		0 ;			\
+	br.sptk		exception_restore ;	\
+}
 
 #define	IVT_ENTRY(name, offset)			\
 	.org	ia64_vector_table + offset;	\
@@ -88,7 +625,7 @@ ivt_##name:
 ia64_vector_table:
 
 IVT_ENTRY(VHPT_Translation, 0x0000)
-	TRAP(0)
+	TRAP(0, cr.ifa)
 IVT_END(VHPT_Translation)
 
 IVT_ENTRY(Instruction_TLB, 0x0400)
@@ -103,8 +640,8 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
 	;; 
 	ld8	r21=[r21]		// check VHPT tag
 	;;
-	cmp.ne	p1,p0=r21,r19
-(p1)	br.dpnt.few 1f
+	cmp.ne	p15,p0=r21,r19
+(p15)	br.dpnt.few 1f
 	;;
 	ld8	r21=[r18]		// read pte
 	mov	pr=r17,0x1ffff
@@ -118,15 +655,15 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
 	;;
 	srlz.d				// serialize
 	;;
-2:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+2:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 3f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 3f		// if not, read next in chain
 	;;
 	ld8	r21=[r20],8		// read pte
 	;; 
@@ -165,7 +702,7 @@ IVT_ENTRY(Instruction_TLB, 0x0400)
 	;;
 	srlz.d
 	;; 
-	TRAP(20)			// Page Not Present trap
+	TRAP(20, cr.ifa)		// Page Not Present trap
 IVT_END(Instruction_TLB)
 
 IVT_ENTRY(Data_TLB, 0x0800)
@@ -180,8 +717,8 @@ IVT_ENTRY(Data_TLB, 0x0800)
 	;; 
 	ld8	r21=[r21]		// check VHPT tag
 	;;
-	cmp.ne	p1,p0=r21,r19
-(p1)	br.dpnt.few 1f
+	cmp.ne	p15,p0=r21,r19
+(p15)	br.dpnt.few 1f
 	;;
 	ld8	r21=[r18]		// read pte
 	mov	pr=r17,0x1ffff
@@ -195,15 +732,15 @@ IVT_ENTRY(Data_TLB, 0x0800)
 	;;
 	srlz.d				// serialize
 	;;
-2:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+2:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 3f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 3f		// if not, read next in chain
 	;;
 	ld8	r21=[r20],8		// read pte
 	;; 
@@ -242,7 +779,7 @@ IVT_ENTRY(Data_TLB, 0x0800)
 	;;
 	srlz.d
 	;; 
-	TRAP(20)			// Page Not Present trap
+	TRAP(20, cr.ifa)		// Page Not Present trap
 IVT_END(Data_TLB)
 
 IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
@@ -251,12 +788,12 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
 	;;
 	extr.u	r17=r16,61,3		// get region number
 	;;
-	cmp.ge	p3,p0=5,r17		// RR0-RR5?
-	cmp.eq	p1,p2=7,r17		// RR7->p1, RR6->p2
-(p3)	br.spnt	9f
+	cmp.ge	p13,p0=5,r17		// RR0-RR5?
+	cmp.eq	p15,p14=7,r17		// RR7->p15, RR6->p14
+(p13)	br.spnt	9f
 	;;
-(p1)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
-(p2)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
+(p15)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
+(p14)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX
 	;;
 	dep	r16=0,r16,50,14		// clear bits above PPN
 	;;
@@ -267,7 +804,7 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00)
 	;;
 	rfi
 9:	mov	pr=r18,0x1ffff		// restore predicates
-	TRAP(3)
+	TRAP(3, cr.ifa)
 IVT_END(Alternate_Instruction_TLB)
 
 IVT_ENTRY(Alternate_Data_TLB, 0x1000)
@@ -276,12 +813,12 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000)
 	;;
 	extr.u	r17=r16,61,3		// get region number
 	;;
-	cmp.ge	p3,p0=5,r17		// RR0-RR5?
-	cmp.eq	p1,p2=7,r17		// RR7->p1, RR6->p2
-(p3)	br.spnt	9f
+	cmp.ge	p13,p0=5,r17		// RR0-RR5?
+	cmp.eq	p15,p14=7,r17		// RR7->p15, RR6->p14
+(p13)	br.spnt	9f
 	;;
-(p1)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
-(p2)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
+(p15)	movl	r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
+(p14)	movl	r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW
 	;;
 	dep	r16=0,r16,50,14		// clear bits above PPN
 	;;
@@ -292,19 +829,92 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000)
 	;;
 	rfi
 9:	mov	pr=r18,0x1ffff		// restore predicates
-	TRAP(4)
+	TRAP(4, cr.ifa)
 IVT_END(Alternate_Data_TLB)
 
 IVT_ENTRY(Data_Nested_TLB, 0x1400)
-	TRAP(5)
+	// See exception_save_restart and exception_restore_restart for the
+	// contexts that may cause a data nested TLB. We can only use the
+	// banked general registers and predicates, but don't use:
+	//	p14 & p15	-	Set in exception save
+	//	r16 & r17	-	Arguments to exception save
+	//	r30		-	Faulting address (modulo page size)
+	// We assume r30 has the virtual addresses that relate to the data
+	// nested TLB fault. The address does not have to be exact, as long
+	// as it's in the same page. We use physical addressing to avoid
+	// double nested faults. Since all virtual addresses we encounter
+	// here are direct mapped region 7 addresses, we have no problem
+	// constructing physical addresses.
+{	.mlx
+	rsm		psr.dt
+	movl		r27=ia64_kptdir
+	;;
+}
+{	.mii
+	srlz.d
+	dep		r27=0,r27,61,3
+	extr.u		r28=r30,PAGE_SHIFT,61-PAGE_SHIFT
+	;;
+}
+{	.mii
+	ld8		r27=[r27]
+	shr.u		r29=r28,PAGE_SHIFT-5	// dir index
+	extr.u		r28=r28,0,PAGE_SHIFT-5	// pte index
+	;;
+}
+{	.mmi
+	shladd		r27=r29,3,r27
+	;;
+	mov		cr.ifa=r30
+	dep		r27=0,r27,61,3
+	;;
+}
+{	.mmi
+	ld8		r27=[r27]
+	mov		r29=rr[r30]
+	shl		r28=r28,5
+	;;
+}
+{	.mii
+	add		r27=r27,r28		// address of pte
+	dep		r29=0,r29,0,2
+	;;
+	dep		r27=0,r27,61,3
+	;;
+}
+{	.mmi
+	ld8		r28=[r27]
+	;;
+	mov		cr.itir=r29
+	or		r28=PTE_D|PTE_A,r28
+	;;
+}
+{	.mlx
+	st8		[r27]=r28
+	movl		r29=exception_save_restart
+	;;
+}
+{	.mmi
+	itc.d		r28
+	;;
+	ssm		psr.dt
+	cmp.eq		p12,p13=r26,r29
+	;;
+}
+{	.mbb
+	srlz.d
+(p12)	br.sptk		exception_save_restart
+(p13)	br.sptk		exception_restore_restart
+	;;
+}
 IVT_END(Data_Nested_TLB)
 
 IVT_ENTRY(Instruction_Key_Miss, 0x1800)
-	TRAP(6)
+	TRAP(6, cr.ifa)
 IVT_END(Instruction_Key_Miss)
 
 IVT_ENTRY(Data_Key_Miss, 0x1c00)
-	TRAP(7)
+	TRAP(7, cr.ifa)
 IVT_END(Data_Key_Miss)
 
 IVT_ENTRY(Dirty_Bit, 0x2000)
@@ -325,15 +935,15 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
 	;;
 	srlz.d				// serialize
 	;;
-1:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+1:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 2f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 2f		// if not, read next in chain
 	;;
 	ld8	r21=[r20]		// read pte
 	mov	r22=PTE_D|PTE_A
@@ -373,7 +983,7 @@ IVT_ENTRY(Dirty_Bit, 0x2000)
 	br.cond.sptk.few 1b		// loop
 
 9:	mov	pr=r17,0x1ffff		// restore predicates
-	TRAP(8)				// die horribly
+	TRAP(8, cr.ifa)			// die horribly
 IVT_END(Dirty_Bit)
 
 IVT_ENTRY(Instruction_Access_Bit, 0x2400)
@@ -394,15 +1004,15 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400)
 	;;
 	srlz.d				// serialize
 	;;
-1:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+1:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 2f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 2f		// if not, read next in chain
 	;;
 	ld8	r21=[r20]		// read pte
 	mov	r22=PTE_A
@@ -442,7 +1052,7 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400)
 	br.cond.sptk.few 1b		// loop
 
 9:	mov	pr=r17,0x1ffff		// restore predicates
-	TRAP(9)
+	TRAP(9, cr.ifa)
 IVT_END(Instruction_Access_Bit)
 
 IVT_ENTRY(Data_Access_Bit, 0x2800)
@@ -463,15 +1073,15 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
 	;;
 	srlz.d				// serialize
 	;;
-1:	cmp.eq	p1,p0=r0,r20		// done?
-(p1)	br.cond.spnt.few 9f		// bail if done
+1:	cmp.eq	p15,p0=r0,r20		// done?
+(p15)	br.cond.spnt.few 9f		// bail if done
 	;;
 	add	r21=16,r20		// tag location
 	;;
 	ld8	r21=[r21]		// read tag
 	;;
-	cmp.ne	p1,p0=r21,r19		// compare tags
-(p1)	br.cond.sptk.few 2f		// if not, read next in chain
+	cmp.ne	p15,p0=r21,r19		// compare tags
+(p15)	br.cond.sptk.few 2f		// if not, read next in chain
 	;;
 	ld8	r21=[r20]		// read pte
 	mov	r22=PTE_A
@@ -511,1344 +1121,283 @@ IVT_ENTRY(Data_Access_Bit, 0x2800)
 	br.cond.sptk.few 1b		// loop
 
 9:	mov	pr=r17,0x1ffff		// restore predicates
-	TRAP(10)
+	TRAP(10, cr.ifa)
 IVT_END(Data_Access_Bit)
 
 IVT_ENTRY(Break_Instruction, 0x2c00)
-	mov	r16=pr			// save pr for a moment
-	mov	r17=cr.iim;;		// read break value
-	mov	r18=0x100000;;		// syscall number
-	cmp.ne	p6,p0=r18,r17;;		// check for syscall
-(p6)	br.dpnt.few 9f
-
-	mov	r17=cr.ipsr;;		// check for user mode
-	extr.u	r17=r17,32,2;;
-	cmp.eq	p6,p0=r0,r17
-(p6)	br.dpnt.few 9f			// trap if kernel mode
-
-	// Note: p6 and p7 are temporaries so we don't need to restore
-	// the value of pr here since the user-mode program assumes
-	// that syscalls only preserve the function-preserved state.
-
-	br.sptk.many	do_syscall
+{	.mib
+	mov		r17=cr.iim
+	mov		r16=ip
+	br.sptk		exception_save
+	;;
+}
+{	.mmi
+	alloc		r15=ar.pfs,0,0,2,0
+	flushrs
+	mov		out0=11
+	;;
+}
+{	.mib
+(p11)	ssm		psr.i
+	add		out1=16,sp
+	br.call.sptk	rp=trap
 	;;
-9:	mov	pr=r16,0x1ffff		// restore pr
-	TRAP(11)
+}
+{	.mfb
+	nop		0
+	nop		0
+	br.sptk		exception_restore
+	;;
+}
 IVT_END(Break_Instruction)
 
 IVT_ENTRY(External_Interrupt, 0x3000)
-	mov	r16=b6			// save user's b6
-1:	mov	r17=ip;;		// construct return address
-	add	r17=2f-1b,r17;;		// for exception_save
-	mov	b6=r17
-	br.sptk.few exception_save	// 'call' exception_save
-
+{	.mib
+	mov		r17=cr.lid	// cr.iim and cr.ifa are undefined.
+	mov		r16=ip
+	br.sptk		exception_save
+	;;
+}
+{	.mmb
 2:	alloc	r14=ar.pfs,0,0,2,0	// make a frame for calling with
-
-	mov	out1=sp;;
-	add	sp=-16,sp;;
+	add	out1=16,sp
+	nop	0
+}
 
 3:	mov	out0=cr.ivr		// find interrupt vector
 	;;
-	cmp.eq	p6,p0=15,out0		// check for spurious vector number
-(p6)	br.dpnt.few exception_restore	// if spurious, we are done
+	cmp.eq	p15,p0=15,out0		// check for spurious vector number
+(p15)	br.dpnt.few exception_restore	// if spurious, we are done
 	;;
 	ssm	psr.i			// re-enable interrupts
-	;;				// now that we are in-progress
-	srlz.d
-	;;
 	br.call.sptk.many rp=interrupt	// call high-level handler
-
+	;;
 	rsm	psr.i			// disable interrupts
 	;;
 	srlz.d
-	;;
 	mov	cr.eoi=r0		// and ack the interrupt
 	;;
 	srlz.d
 	br.sptk.few 3b			// loop for more
+	;;
 IVT_END(External_Interrupt)
 
 IVT_ENTRY(Reserved_3400, 0x3400)
-	TRAP(13)
+	TRAP(13, cr.ifa)
 IVT_END(Reserved_3400)
 
 IVT_ENTRY(Reserved_3800, 0x3800)
-	TRAP(14)
+	TRAP(14, cr.ifa)
 IVT_END(Reserved_3800)
 
 IVT_ENTRY(Reserved_3c00, 0x3c00)
-	TRAP(15)
+	TRAP(15, cr.ifa)
 IVT_END(Reserved_3c00)
 
 IVT_ENTRY(Reserved_4000, 0x4000)
-	TRAP(16)
+	TRAP(16, cr.ifa)
 IVT_END(Reserved_4000)
 
 IVT_ENTRY(Reserved_4400, 0x4400)
-	TRAP(17)
+	TRAP(17, cr.ifa)
 IVT_END(Reserved_4400)
 
 IVT_ENTRY(Reserved_4800, 0x4800)
-	TRAP(18)
+	TRAP(18, cr.ifa)
 IVT_END(Reserved_4800)
 
 IVT_ENTRY(Reserved_4c00, 0x4c00)
-	TRAP(19)
+	TRAP(19, cr.ifa)
 IVT_END(Reserved_4c00)
 
 IVT_ENTRY(Page_Not_Present, 0x5000)
-	TRAP(20)
+	TRAP(20, cr.ifa)
 IVT_END(Page_Not_Present)
 
 IVT_ENTRY(Key_Permission, 0x5100)
-	TRAP(21)
+	TRAP(21, cr.ifa)
 IVT_END(Key_Permission)
 
 IVT_ENTRY(Instruction_Access_Rights, 0x5200)
-	TRAP(22)
+	TRAP(22, cr.ifa)
 IVT_END(Instruction_Access_Rights)
 
 IVT_ENTRY(Data_Access_Rights, 0x5300)
-	TRAP(23)
+	TRAP(23, cr.ifa)
 IVT_END(Data_Access_Rights)
 
 IVT_ENTRY(General_Exception, 0x5400)
-	TRAP(24)
+	TRAP(24, cr.ifa)
 IVT_END(General_Exception)
 
 IVT_ENTRY(Disabled_FP_Register, 0x5500)
-	TRAP(25)
+	TRAP(25, cr.ifa)
 IVT_END(Disabled_FP_Register)
 
 IVT_ENTRY(NaT_Consumption, 0x5600)
-	TRAP(26)
+	TRAP(26, cr.ifa)
 IVT_END(NaT_Consumption)
 
 IVT_ENTRY(Speculation, 0x5700)
-	TRAP(27)
+	TRAP(27, cr.iim)
 IVT_END(Speculation)
 
 IVT_ENTRY(Reserved_5800, 0x5800)
-	TRAP(28)
+	TRAP(28, cr.ifa)
 IVT_END(Reserved_5800)
 
 IVT_ENTRY(Debug, 0x5900)
-	TRAP(29)
+	TRAP(29, cr.ifa)
 IVT_END(Debug)
 
 IVT_ENTRY(Unaligned_Reference, 0x5a00)
-	TRAP(30)
+	TRAP(30, cr.ifa)
 IVT_END(Unaligned_Reference)
 
 IVT_ENTRY(Unsupported_Data_Reference, 0x5b00)
-	TRAP(31)
+	TRAP(31, cr.ifa)
 IVT_END(Unsupported_Data_Reference)
 
 IVT_ENTRY(Floating_Point_Fault, 0x5c00)
-	TRAP(32)
+	TRAP(32, cr.ifa)
 IVT_END(Floating_Point_Fault)
 
 IVT_ENTRY(Floating_Point_Trap, 0x5d00)
-	TRAP(33)
+	TRAP(33, cr.ifa)
 IVT_END(Floating_Point_Trap)
 
 IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00)
-	TRAP(34)
+	TRAP(34, cr.ifa)
 IVT_END(Lower_Privilege_Transfer_Trap)
 
 IVT_ENTRY(Taken_Branch_Trap, 0x5f00)
-	TRAP(35)
+	TRAP(35, cr.ifa)
 IVT_END(Taken_Branch_Trap)
 
 IVT_ENTRY(Single_Step_Trap, 0x6000)
-	TRAP(36)
+	TRAP(36, cr.ifa)
 IVT_END(Single_Step_Trap)
 
 IVT_ENTRY(Reserved_6100, 0x6100)
-	TRAP(37)
+	TRAP(37, cr.ifa)
 IVT_END(Reserved_6100)
 
 IVT_ENTRY(Reserved_6200, 0x6200)
-	TRAP(38)
+	TRAP(38, cr.ifa)
 IVT_END(Reserved_6200)
 
 IVT_ENTRY(Reserved_6300, 0x6300)
-	TRAP(39)
+	TRAP(39, cr.ifa)
 IVT_END(Reserved_6300)
 
 IVT_ENTRY(Reserved_6400, 0x6400)
-	TRAP(40)
+	TRAP(40, cr.ifa)
 IVT_END(Reserved_6400)
 
 IVT_ENTRY(Reserved_6500, 0x6500)
-	TRAP(41)
+	TRAP(41, cr.ifa)
 IVT_END(Reserved_6500)
 
 IVT_ENTRY(Reserved_6600, 0x6600)
-	TRAP(42)
+	TRAP(42, cr.ifa)
 IVT_END(Reserved_6600)
 
 IVT_ENTRY(Reserved_6700, 0x6700)
-	TRAP(43)
+	TRAP(43, cr.ifa)
 IVT_END(Reserved_6700)
 
 IVT_ENTRY(Reserved_6800, 0x6800)
-	TRAP(44)
+	TRAP(44, cr.ifa)
 IVT_END(Reserved_6800)
 
 IVT_ENTRY(IA_32_Exception, 0x6900)
-	TRAP(45)
+	TRAP(45, cr.ifa)
 IVT_END(IA_32_Exception)
 
 IVT_ENTRY(IA_32_Intercept, 0x6a00)
-	TRAP(46)
+	TRAP(46, cr.iim)
 IVT_END(IA_32_Intercept)
 
 IVT_ENTRY(IA_32_Interrupt, 0x6b00)
-	TRAP(47)
+	TRAP(47, cr.ifa)
 IVT_END(IA_32_Interrupt)
 
 IVT_ENTRY(Reserved_6c00, 0x6c00)
-	TRAP(48)
+	TRAP(48, cr.ifa)
 IVT_END(Reserved_6c00)
 
 IVT_ENTRY(Reserved_6d00, 0x6d00)
-	TRAP(49)
+	TRAP(49, cr.ifa)
 IVT_END(Reserved_6d00)
 
 IVT_ENTRY(Reserved_6e00, 0x6e00)
-	TRAP(50)
+	TRAP(50, cr.ifa)
 IVT_END(Reserved_6e00)
 
 IVT_ENTRY(Reserved_6f00, 0x6f00)
-	TRAP(51)
+	TRAP(51, cr.ifa)
 IVT_END(Reserved_6f00)
 
 IVT_ENTRY(Reserved_7000, 0x7000)
-	TRAP(52)
+	TRAP(52, cr.ifa)
 IVT_END(Reserved_7000)
 
 IVT_ENTRY(Reserved_7100, 0x7100)
-	TRAP(53)
+	TRAP(53, cr.ifa)
 IVT_END(Reserved_7100)
 
 IVT_ENTRY(Reserved_7200, 0x7200)
-	TRAP(54)
+	TRAP(54, cr.ifa)
 IVT_END(Reserved_7200)
 
 IVT_ENTRY(Reserved_7300, 0x7300)
-	TRAP(55)
+	TRAP(55, cr.ifa)
 IVT_END(Reserved_7300)
 
 IVT_ENTRY(Reserved_7400, 0x7400)
-	TRAP(56)
+	TRAP(56, cr.ifa)
 IVT_END(Reserved_7400)
 
 IVT_ENTRY(Reserved_7500, 0x7500)
-	TRAP(57)
+	TRAP(57, cr.ifa)
 IVT_END(Reserved_7500)
 
 IVT_ENTRY(Reserved_7600, 0x7600)
-	TRAP(58)
+	TRAP(58, cr.ifa)
 IVT_END(Reserved_7600)
 
 IVT_ENTRY(Reserved_7700, 0x7700)
-	TRAP(59)
+	TRAP(59, cr.ifa)
 IVT_END(Reserved_7700)
 
 IVT_ENTRY(Reserved_7800, 0x7800)
-	TRAP(60)
+	TRAP(60, cr.ifa)
 IVT_END(Reserved_7800)
 
 IVT_ENTRY(Reserved_7900, 0x7900)
-	TRAP(61)
+	TRAP(61, cr.ifa)
 IVT_END(Reserved_7900)
 
 IVT_ENTRY(Reserved_7a00, 0x7a00)
-	TRAP(62)
+	TRAP(62, cr.ifa)
 IVT_END(Reserved_7a00)
 
 IVT_ENTRY(Reserved_7b00, 0x7b00)
-	TRAP(63)
+	TRAP(63, cr.ifa)
 IVT_END(Reserved_7b00)
 
 IVT_ENTRY(Reserved_7c00, 0x7c00)
-	TRAP(64)
+	TRAP(64, cr.ifa)
 IVT_END(Reserved_7c00)
 
 IVT_ENTRY(Reserved_7d00, 0x7d00)
-	TRAP(65)
+	TRAP(65, cr.ifa)
 IVT_END(Reserved_7d00)
 
 IVT_ENTRY(Reserved_7e00, 0x7e00)
-	TRAP(66)
+	TRAP(66, cr.ifa)
 IVT_END(Reserved_7e00)
 
 IVT_ENTRY(Reserved_7f00, 0x7f00)
-	TRAP(67)
+	TRAP(67, cr.ifa)
 IVT_END(Reserved_7f00)
-
-	.text
-
-/*
- * exception_restore:	restore interrupted state
- *
- * Arguments:
- *	sp+16	trapframe pointer
- *	r4	ar.pfs before the alloc in TRAP()
- *
- */
-ENTRY(exception_restore, 0)
-{	.mfi
-	alloc	r14=ar.pfs,0,0,1,0	// in case we call ast()
-	nop	1
-	add	r3=TF_CR_IPSR+16,sp
-	;;
-}
-{	.mmi
-	ld8	r30=[r3]		// ipsr
-	;;
-	nop	2
-	extr.u	r16=r30,32,2		// extract ipsr.cpl
-	;;
-}
-{	.mfb
-	cmp.eq	p1,p2=r0,r16		// test for return to kernel mode
-	nop	3
-(p1)	br.cond.dpnt 2f			// no ast check for returns to kernel
-}
-3:
-{	.mmi
-	add	r3=PC_CURTHREAD,r13	// &curthread
-	;;
-	ld8	r3=[r3]			// curthread
-	add	r2=(TDF_ASTPENDING|TDF_NEEDRESCHED),r0
-	;;
-}
-{	.mmb
-	mov	r15=psr			// save interrupt enable status
-	nop	4
-	;;
-}
-{	.mmi
-	;;
-	rsm	psr.i			// disable interrupts
-	add	r3=TD_FLAGS,r3		// &curthread->td_flags
-	;;
-}
-{	.mmi
-	ld4	r14=[r3]		// fetch curthread->td_flags
-	;;
-	and	r14=r2,r14	    // flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)
-	nop	5
-	;;
-}
-{	.mfb
-	cmp4.eq	p6,p7=r0,r14		//  == 0 ?
-	nop	6
-(p6)	br.cond.dptk	2f
-	;;
-}
-{	.mmi
-	mov	psr.l=r15		// restore interrups
-	;;
-	srlz.d
-	add	out0=16,sp		// trapframe argument to ast()
-}
-{	.bbb
-	br.call.sptk.many rp=ast	// note: p1, p2 preserved
-	br.sptk	3b
-	nop	7
-}
-2:
-{	.mmi
-	rsm	psr.ic|psr.dt|psr.i	// disable interrupt collection and vm
-	;;
-	srlz.i
-	add	r3=16,sp
-	;;
-}
-{	.mmi
-(p2)	add	r16=SIZEOF_TRAPFRAME+16,sp  // restore ar.k6 (kernel sp)
-	;;
-(p2)	mov	ar.k6=r16
-	dep	r3=0,r3,61,3		// physical address
-	;; 
-}
-{	.mmi
-	add	r1=SIZEOF_TRAPFRAME-16,r3 // r1=&tf_f[FRAME_F15]
-	;;
-	ldf.fill f15=[r1],-32		// r1=&tf_f[FRAME_F13]
-	add	r2=SIZEOF_TRAPFRAME-32,r3 // r2=&tf_f[FRAME_F14]
-	;;
-}
-{	.mmb
-	ldf.fill f14=[r2],-32		// r2=&tf_f[FRAME_F12]
-	ldf.fill f13=[r1],-32		// r1=&tf_f[FRAME_F11]
-	nop	8
-	;;
-}
-{	.mmi
-	ldf.fill f12=[r2],-32		// r2=&tf_f[FRAME_F10]
-	ldf.fill f11=[r1],-32		// r1=&tf_f[FRAME_F9]
-	nop	9
-	;;
-}
-{	.mmb
-	ldf.fill f10=[r2],-32		// r2=&tf_f[FRAME_F8]
-	ldf.fill f9=[r1],-32		// r1=&tf_f[FRAME_F7]
-	nop	10
-	;;
-}
-{	.mmi
-	ldf.fill f8=[r2],-32		// r2=&tf_f[FRAME_F6]
-	ldf.fill f7=[r1],-24		// r1=&tf_r[FRAME_R31]
-	nop	11
-	;;
-}
-{	.mmb
-	ldf.fill f6=[r2],-16		// r2=&tf_r[FRAME_R30]
-	ld8.fill r31=[r1],-16		// r1=&tf_r[FRAME_R29]
-	nop	12
-	;;
-}
-{	.mmi
-	ld8.fill r30=[r2],-16		// r2=&tf_r[FRAME_R28]
-	ld8.fill r29=[r1],-16		// r1=&tf_r[FRAME_R27]
-	nop	13
-	;;
-}
-{	.mmb
-	ld8.fill r28=[r2],-16		// r2=&tf_r[FRAME_R26]
-	ld8.fill r27=[r1],-16		// r1=&tf_r[FRAME_R25]
-	nop	14
-	;;
-}
-{	.mmi
-	ld8.fill r26=[r2],-16		// r2=&tf_r[FRAME_R24]
-	ld8.fill r25=[r1],-16		// r1=&tf_r[FRAME_R23]
-	nop	15
-	;;
-}
-{	.mmb
-	ld8.fill r24=[r2],-16		// r2=&tf_r[FRAME_R22]
-	ld8.fill r23=[r1],-16		// r1=&tf_r[FRAME_R21]
-	nop	16
-	;;
-}
-{	.mmi
-	ld8.fill r22=[r2],-16		// r2=&tf_r[FRAME_R20]
-	ld8.fill r21=[r1],-16		// r1=&tf_r[FRAME_R19]
-	nop	17
-	;;
-}
-{	.mmb
-	ld8.fill r20=[r2],-16		// r2=&tf_r[FRAME_R18]
-	ld8.fill r19=[r1],-16		// r1=&tf_r[FRAME_R17]
-	nop	18
-	;;
-}
-{	.mmi
-	ld8.fill r18=[r2],-16		// r2=&tf_r[FRAME_R16]
-	ld8.fill r17=[r1],-16		// r1=&tf_r[FRAME_R15]
-	nop	19
-	;;
-}
-{	.mfb
-	ld8.fill r16=[r2],-16		// r2=&tf_r[FRAME_R14]
-	nop	20
-	bsw.0				// switch to bank 0
-	;;
-}
-{	.mmi
-	ld8.fill r15=[r1],-16		// r1=&tf_r[FRAME_R13]
-	ld8.fill r14=[r2],-16		// r2=&tf_r[FRAME_R12]
-	nop	21
-	;;
-}
-	// Don't restore r13 if returning to kernel
-{	.mmi
-	.pred.rel.mutex p1,p2
-(p2)	ld8.fill r13=[r1],-16		// r1=&tf_r[FRAME_R11]
-	ld8.fill r12=[r2],-16		// r2=&tf_r[FRAME_R10]
-(p1)	add	r1=-16,r1		// r1=&tf_r[FRAME_R11]
-	;;
-}
-{	.mmb
-	ld8.fill r11=[r1],-16		// r1=&tf_r[FRAME_R9]
-	ld8.fill r10=[r2],-16		// r2=&tf_r[FRAME_R8]
-	nop	22
-	;;
-}
-{	.mmi
-	ld8.fill r9=[r1],-16		// r1=&tf_r[FRAME_R7]
-	ld8.fill r8=[r2],-16		// r2=&tf_r[FRAME_R6]
-	nop	23
-	;;
-}
-{	.mmb
-	ld8.fill r7=[r1],-16		// r1=&tf_r[FRAME_R5]
-	ld8.fill r6=[r2],-16		// r2=&tf_r[FRAME_R4]
-	nop	24
-	;;
-}
-{	.mmi
-	ld8.fill r5=[r1],-16		// r1=&tf_r[FRAME_R3]
-	ld8.fill r4=[r2],-16		// r2=&tf_r[FRAME_R2]
-	nop	25
-	;;
-}
-{	.mmb
-	ld8.fill r3=[r1],-16		// r1=&tf_r[FRAME_R1]
-	ld8.fill r23=[r2],-16		// r2=&tf_b[7]
-	nop	26
-	;;
-}
-{	.mmi
-	ld8.fill r24=[r1],-16		// r1=&tf_b[6]
-	ld8	r16=[r2],-16		// r16=b7, r2=&tf_b[5]
-	nop	27
-	;;
-}
-{	.mmi
-	ld8	r17=[r1],-16		// r17=b6, r1=&tf_b[4]
-	ld8	r18=[r2],-16		// r18=b5, r2=&tf_b[3]
-	mov	b7=r16
-	;;
-}
-{	.mmi
-	ld8	r16=[r1],-16		// r16=b4, r1=&tf_b[2]
-	ld8	r19=[r2],-16		// r19=b3, r2=&tf_b[1]
-	mov	b6=r17
-	;;
-}
-{	.mii
-	ld8	r17=[r1],-16		// r17=b2, r1=&tf_b[0]
-	mov	b5=r18
-	mov	b4=r16
-	;;
-}
-{	.mii
-	ld8	r16=[r2],-16		// r16=b1, r2=&tf_ar_ec
-	mov	b3=r19
-	mov	b2=r17
-	;;
-}
-{	.mmi
-	ld8	r17=[r1],-16		// r17=b0, r1=&tf_ar_lc
-	ld8	r18=[r2],-16		// r18=ar.ec, r2=&tf_ar_fptr
-	mov	b1=r16
-	;;
-}
-{	.mmi
-	ld8	r16=[r1],-16		// r16=ar.lc, r1=&tf_ar_ccv
-	ld8	r19=[r2],-16		// r19=ar.fpsr, r1=&tf_ar_unat
-	mov	b0=r17
-	;;
-}
-{	.mmi
-	ld8	r17=[r1],-16		// r17=ar.ccv, r1=&tf_ndirty
-	mov	ar.fpsr=r19
-	mov	ar.ec=r18
-	;;
-}
-{	.mmi
-	ld8	r18=[r2],-16		// r18=ar.unat, r2=&tf_ar_rnat
-	mov	ar.ccv=r17
-	mov	ar.lc=r16
-	;;
-}
-{	.mmb
-	ld8	r27=[r1],-16	// r1=&tf_ar_bspstore
-	ld8	r21=[r2],-16		// r2=&tf_cr_ifs
-	nop	28
-	;;
-}
-{	.mmi
-	mov	ar.unat=r18
-	ld8	r22=[r1],-16	// r1=&tf_ar_pfs
-	nop	29
-}
-{	.mfb
-	ld8	r25=[r2],-16		// r2=&tf_ar_rsc
-	nop	30
-(p1)	br.cond.dpnt.few 1f		// don't switch bs if kernel
-	;;
-}
-{	.mmi
-	alloc	r16=ar.pfs,0,0,0,0	// discard current frame
-	;;
-	nop	31
-	shl	r16=r27,16		// value for ar.rsc
-	;;
-}
-{	.mmi
-	mov	ar.rsc=r16		// setup for loadrs
-	;;
-	loadrs				// load user regs from kernel bs
-	nop	32
-	;;
-}
-{	.mmi
-	mov	ar.bspstore=r22
-	;;
-	mov	ar.rnat=r21
-	nop	33
-	;;
-}
-1:
-{	.mmb
-	ld8	r19=[r1],-16		// r1=&tf_pr
-	ld8	r20=[r2],-16		// r2=&tf_cr_ifa
-	nop	34
-	;;
-}
-{	.mmi
-	ld8	r27=[r1],-16		// r1=&tf_cr_isr
-	ld8	r28=[r2],-16		// r2=&tf_cr_ipsr
-	mov	ar.pfs=r19
-	;;
-}
-{	.mmi
-	ld8	r29=[r1],-16		// r1=&tf_cr_iip
-	ld8	r30=[r2]
-	mov	pr=r27,0x1ffff
-	;;
-}
-{	.mmi
-	ld8	r31=[r1]
-	mov	cr.ifs=r25
-	mov	r2=r23
-	;;
-}
-{	.mmi
-	mov	cr.ifa=r28
-	mov	cr.iip=r31
-	mov	r1=r24
-	;;
-}
-{	.mmi
-	mov	cr.ipsr=r30
-	mov	ar.rsc=r20
-	nop	35
-	;;
-}
-{	.bbb
-	nop	36
-	nop	37
-	rfi
-	;;
-}
-END(exception_restore)
-	
-
-/*
- * exception_save: save interrupted state
- *
- * Arguments:
- *	b6	return address
- *	r16	saved b6
- *
- * Return:
- *	r14	cr.iim value for break traps
- *	sp	kernel stack pointer
- *	p1	true if user mode
- *	p2	true if kernel mode
- *	p3	true if interrupts were enabled
- */
-ENTRY(exception_save, 0)
-{	.mmi
-	rsm	psr.dt			// turn off data translations
-	;;
-	srlz.d				// serialize
-	mov	r27=pr
-}
-{	.mmi
-	mov	r30=cr.ipsr
-	;;
-	mov	r31=cr.iip
-	tbit.nz	p3,p0=r30,14		// check for interrupt enable state
-}
-{	.mmi
-	mov	r29=cr.isr
-	;; 
-	mov	r26=sp			// save sp
-	extr.u	r17=r30,32,2		// extract ipsr.cpl
-	;;
-}
-{	.mmi
-	cmp.eq	p1,p2=r0,r17		// test for kernel mode
-	;;
-(p2)	mov	sp=ar.k6		// and switch to kernel stack
-	mov	r24=r1
-	;;
-}
-{	.mii
-	mov	r28=cr.ifa
-	add	sp=-SIZEOF_TRAPFRAME,sp	// reserve trapframe
-	;;
-	dep	r1=0,sp,61,3		// r1=&tf_flags
-	;;
-}
-{	.mmi
-	st8	[r1]=r0,8		// zero flags, r1=&tf_cr_iip
-	;;
-	mov	r23=r2
-	add	r2=8,r1			// r2=&tf_cr_ipsr
-	;;
-}
-{	.mmb
-	st8	[r1]=r31,16		// r1=&tf_cr_isr
-	st8	[r2]=r30,16		// r2=&tf_cr_ifa
-	nop	1
-	;;
-}
-{	.mmb
-	st8	[r1]=r29,16		// r1=&tf_pr
-	st8	[r2]=r28,16		// r2=&tf_ar_rsc
-	nop	2
-	;;
-}
-{	.mmi
-	st8	[r1]=r27,16		// r1=&tf_cr_pfs
-	mov	r20=ar.rsc
-	mov	r19=ar.pfs
-	;;
-}
-{	.mmb
-	st8	[r2]=r20,16		// r2=&tf_cr_ifs
-	st8	[r1]=r19,16		// r1=&tf_ar_bspstore
-	cover
-	;;
-}
-{	.mmi
-	mov	ar.rsc=0
-	;;
-	mov	r22=ar.bspstore
-	;;
-}
-{	.mmi
-	mov	r25=cr.ifs
-	mov	r21=ar.rnat
-(p1)	mov	r31=r22		// so we can figure out ndirty
-	;;
-}
-{	.mmb
-(p2)	mov	r31=ar.k5		// kernel backing store
-	st8	[r2]=r25,16		// r2=&tf_ar_rnat
-	nop	3
-	;;
-}
-{	.mmi
-	st8	[r1]=r22,16	// r1=&tf_ndirty
-(p2)	mov	ar.bspstore=r31		// switch bspstore
-	nop	4
-	;;
-}
-{	.mmi
-	mov	r17=ar.bsp
-	;;
-	st8	[r2]=r21,16		// r2=&tf_ar_unat
-	sub	r17=r17,r31		// ndirty (in bytes)
-	;;
-}
-{	.mmi
-	st8	[r1]=r17,16		// r1=&tf_ar_ccv
-	mov	ar.rsc=3		// switch RSE back on
-	mov	r18=ar.lc
-}
-{	.mmi
-	mov	r31=ar.unat
-	;;
-	st8	[r2]=r31,16		// r2=&tf_ar_fpsr
-	mov	r19=ar.ec
-}
-{	.mmi
-	mov	r17=ar.ccv
-	;;
-	st8	[r1]=r17,16		// r1=&tf_ar_lc
-	nop	5
-	;;
-}
-{	.mmi
-	mov	r31=ar.fpsr
-	;;
-	st8	[r2]=r31,16		// r2=&tf_ar_ec
-	mov	r30=b0
-	;;
-}
-{	.mmi
-	st8	[r1]=r18,16		// r1=&tf_b[0]
-	;;
-	st8	[r2]=r19,16		// r2=&tf_b[1]
-	mov	r31=b1
-}
-{	.mmi
-	st8	[r1]=r30,16		// r1=&tf_b[2]
-	;;
-	st8	[r2]=r31,16		// r2=&tf_b[3]
-	mov	r31=b2
-	;;
-}
-{	.mii
-	st8	[r1]=r31,16		// r1=&tf_b[4]
-	mov	r17=b3
-	;;
-	mov	r18=b4
-}
-{	.mmi
-	st8	[r2]=r17,16		// r2=&tf_b[5]
-	;; 
-	st8	[r1]=r18,16		// r1=&tf_b[6]
-	mov	r31=b5
-	;;
-}
-{	.mii
-	st8	[r2]=r31,16		// r2=&tf_b[7]
-	mov	r18=b7
-}
-{	.mmi
-	st8	[r1]=r16,16		// r1=&tf_r[FRAME_R1]
-	;; 
-	st8	[r2]=r18,16		// r2=&tf_r[FRAME_R2]
-	nop	7
-	;;
-}
-{	.mmb
-	.mem.offset 0,0
-	st8.spill [r1]=r24,16		// r1=&tf_r[FRAME_R3]
-	.mem.offset 8,0
-	st8.spill [r2]=r23,16		// r2=&tf_r[FRAME_R4]
-	nop	8
-	;;
-}
-{	.mmi
-	.mem.offset 16,0
-	st8.spill [r1]=r3,16		// r1=&tf_r[FRAME_R5]
-	.mem.offset 32,0
-	st8.spill [r2]=r4,16		// r2=&tf_r[FRAME_R6]
-	nop	9
-	;;
-}
-{	.mmb
-	.mem.offset 48,0
-	st8.spill [r1]=r5,16		// r1=&tf_r[FRAME_R7]
-	.mem.offset 64,0
-	st8.spill [r2]=r6,16		// r2=&tf_r[FRAME_R8]
-	nop	10
-	;;
-}
-{	.mmi
-	.mem.offset 80,0
-	st8.spill [r1]=r7,16		// r1=&tf_r[FRAME_R9]
-	.mem.offset 96,0
-	st8.spill [r2]=r8,16		// r2=&tf_r[FRAME_R10]
-	nop	11
-	;;
-}
-{	.mmb
-	.mem.offset 112,0
-	st8.spill [r1]=r9,16		// r1=&tf_r[FRAME_R11]
-	.mem.offset 128,0
-	st8.spill [r2]=r10,16		// r2=&tf_r[FRAME_SP]
-	nop	12
-	;;
-}
-{	.mmi
-	.mem.offset 144,0
-	st8.spill [r1]=r11,16		// r1=&tf_r[FRAME_R13]
-	.mem.offset 160,0
-	st8.spill [r2]=r26,16		// r2=&tf_r[FRAME_R14]
-	nop	13
-	;;
-}
-{	.mmb
-	.mem.offset 176,0
-	st8.spill [r1]=r13,16		// r1=&tf_r[FRAME_R15]
-	.mem.offset 192,0
-	st8.spill [r2]=r14,16		// r2=&tf_r[FRAME_R16]
-	nop	14
-	;;
-}
-{	.mfb
-	.mem.offset 208,0
-	st8.spill [r1]=r15,16		// r1=&tf_r[FRAME_R17]
-	nop	15
-	bsw.1				// switch to bank 1
-	;;
-}
-{	.mmi
-	.mem.offset 224,0
-	st8.spill [r2]=r16,16		// r2=&tf_r[FRAME_R18]
-	.mem.offset 240,0
-	st8.spill [r1]=r17,16		// r1=&tf_r[FRAME_R19]
-	nop	16
-	;;
-}
-{	.mmb
-	.mem.offset 256,0
-	st8.spill [r2]=r18,16		// r2=&tf_r[FRAME_R20]
-	.mem.offset 272,0
-	st8.spill [r1]=r19,16		// r1=&tf_r[FRAME_R21]
-	nop	17
-	;;
-}
-{	.mmi
-	.mem.offset 288,0
-	st8.spill [r2]=r20,16		// r2=&tf_r[FRAME_R22]
-	.mem.offset 304,0
-	st8.spill [r1]=r21,16		// r1=&tf_r[FRAME_R23]
-	nop	18
-	;;
-}
-{	.mmb
-	.mem.offset 320,0
-	st8.spill [r2]=r22,16		// r2=&tf_r[FRAME_R24]
-	.mem.offset 336,0
-	st8.spill [r1]=r23,16		// r1=&tf_r[FRAME_R25]
-	nop	19
-	;;
-}
-{	.mmi
-	.mem.offset 352,0
-	st8.spill [r2]=r24,16		// r2=&tf_r[FRAME_R26]
-	.mem.offset 368,0
-	st8.spill [r1]=r25,16		// r1=&tf_r[FRAME_R27]
-	nop	20
-	;;
-}
-{	.mmb
-	.mem.offset 384,0
-	st8.spill [r2]=r26,16		// r2=&tf_r[FRAME_R28]
-	.mem.offset 400,0
-	st8.spill [r1]=r27,16		// r1=&tf_r[FRAME_R29]
-	nop	21
-	;;
-}
-{	.mmi
-	.mem.offset 416,0
-	st8.spill [r2]=r28,16		// r2=&tf_r[FRAME_R30]
-	.mem.offset 432,0
-	st8.spill [r1]=r29,16		// r1=&tf_r[FRAME_R31]
-	nop	22
-	;;
-}
-{	.mmb
-	.mem.offset 448,0
-	st8.spill [r2]=r30,16		// r2=&tf_f[FRAME_F6]
-	.mem.offset 464,0
-	st8.spill [r1]=r31,24		// r1=&tf_f[FRAME_F7]
-	nop	23
-	;;
-}
-{	.mmi
-	stf.spill [r2]=f6,32		// r2=&tf_f[FRAME_F8]
-	stf.spill [r1]=f7,32		// r1=&tf_f[FRAME_F9]
-	nop	24
-	;;
-}
-{	.mmb
-	stf.spill [r2]=f8,32		// r2=&tf_f[FRAME_F10]
-	stf.spill [r1]=f9,32		// r1=&tf_f[FRAME_F11]
-	nop	25
-	;;
-}
-{	.mmi
-	stf.spill [r2]=f10,32		// r2=&tf_f[FRAME_F12]
-	stf.spill [r1]=f11,32		// r1=&tf_f[FRAME_F13]
-	nop	26
-	;;
-}
-{	.mmb
-	stf.spill [r2]=f12,32		// r2=&tf_f[FRAME_F14]
-	stf.spill [r1]=f13,32		// r1=&tf_f[FRAME_F15]
-	nop	27
-	;;
-}
-{	.mmi
-	stf.spill [r2]=f14		//
-	stf.spill [r1]=f15		//
-	nop	28
-	;;
-}
-{	.mlx
-	mov	r14=cr.iim		// break immediate
-	movl	r1=__gp			// kernel globals
-}
-{	.mmi
-	ssm	psr.ic|psr.dt		// enable interrupts & translation
-	;;
-	srlz.i				// serialize
-	nop	29
-	;;
-}
-{	.mfb
-	mov	r13=ar.k4		// processor globals
-	nop	30
-	br.sptk.few b6			// not br.ret - we were not br.call'ed
-	;;
-}
-END(exception_save)
-	
-/*
- * System call entry point (via Break Instruction vector).
- *
- * Arguments:
- *	r15		System call number
- *	out0-out7	System call arguments
- */
-ENTRY(do_syscall, 0)
-	.prologue
-	.unwabi	@svr4, 'I'
-	.save	rp,r0
-	.body
-	// Save minimal state for syscall.
-	// We need to save enough state so that sendsig doesn't
-	// trash things if we take a signal during the system call.
-	// Essentially we need to save all the function-preserved
-	// state. Note that if we don't take a signal, we don't need
-	// to restore much of that state on the way out. Note also
-	// that when we save r4-r7 we spill their NaT bits into
-	// ar.unat. This register is preserved by the call to
-	// syscall() and if a full restore is needed,
-	// exception_restore will recover the NaT bits from ar.unat.
-	// The function-preserved state (including syscall number) is:
-	//
-	//	r1,r4-r7,sp,r15
-	//	f16-f31
-	//	p1-p5,p16-p63
-	//	b0-b5
-	//	various ar's
-	//
-{ .mmi					// start reading high latency regs
-	mov	r16=cr.ipsr		// (13)
-	mov.m	r17=ar.rsc		// (13)
-	mov	r18=sp			// save user sp
-	;;
-} { .mmi
-	mov	sp=ar.k6		// (13) kernel sp
-	mov	r19=cr.isr		// (13)
-	nop.i	0
-} { .mmi
-	mov.m	ar.rsc=0
-	;; 
-	mov.m	r20=ar.bspstore		// (13)
-	nop.i	0
-} { .mmi
-	mov.m	r21=ar.k5		// (13)
-	mov.m	r22=ar.rnat		// (6)
-	nop.i	0
-} { .mmi
-	mov.m	r23=ar.unat		// (6)
-	rsm	psr.dt			// (5) physical addressing
-} { .mii
-	mov	r24=cr.iip		// (2)
-	mov.i	r25=ar.pfs		// (2)
-	add	sp=-SIZEOF_TRAPFRAME,sp // reserve trapframe
-	;; 
-} { .mii
-	addl	r27=FRAME_SYSCALL,r0	// (1)
-	mov	r26=pr			// (2)
-	dep	r30=0,sp,61,3		// physical address
-} { .mmi
-	srlz.d				// serialize psr.dt
-	;; 
-	add	r31=8,r30		// secondary pointer
-	;; 
-} { .mmi
-	st8	[r30]=r27,16		// tf_flags
-	st8	[r31]=r24,16		// save cr.iip
-	mov	r28=b0
-	;;
-} { .mmi
-	st8	[r30]=r16,24		// save cr.ipsr, skip to pr
-	st8	[r31]=r19,24		// save cr.isr, skip to ar.rsc
-	mov	r24=b1
-	;;
-} { .mmi
-	st8	[r30]=r26,16		// save pr, skip to ar.pfs
-	st8	[r31]=r17,24		// save ar.rsc, skip to ar.bspstore
-	mov	r27=b2
-	;;
-} { .mii
-	st8	[r30]=r25,24		// save ar.pfs, skip to ar.rnat
-	mov	r16=b3
-	mov	r17=b4
-	;;
-} { .mmi
-	st8	[r31]=r20,24		// save ar.bspstore, skip to ar.unat
-	mov.m	ar.bspstore=r21		// switch to kernel backing store
-	mov	r29=b5
-	;;
-} { .mmi
-	mov.m	r20=ar.ccv
-	mov.m	r21=ar.fpsr
-	nop.i	0
-	;; 
-} { .mmi
-	st8	[r30]=r22,24		// save ar.rnat, skip to ar.ccv
-	st8	[r31]=r23,16		// save ar.unat, skip to ar.fpsr
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r20,32		// save ar.ccv, skip to b0
-	st8	[r31]=r21,32		// save ar.fpsr, skip to b1
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r28,16		// save b0, skip to b2
-	st8	[r31]=r24,16		// save b1, skip to b3
-	nop.i	0
-	;; 
-} { .mmi
-	st8	[r30]=r27,16		// save b2, skip to b4
-	st8	[r31]=r16,16		// save b3, skip to b5
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r17,TF_R_R1-(TF_B+4*8) // save b4, skip to r1
-	st8	[r31]=r29,TF_R_R4-(TF_B+5*8) // save b5, skip to r4
-	nop.i	0
-	;;
-} { .mmi
-	st8	[r30]=r1,TF_R_R5-TF_R_R1 // save r1, skip to r5
-	.mem.offset 8,0
-	st8.spill [r31]=r4,16		// save r4, skip to r6
-	nop.i	0
-	;;
-} { .mmi
-	.mem.offset 0,0
-	st8.spill [r30]=r5,16		// save r5, skip to r7
-	.mem.offset 8,0
-	st8.spill [r31]=r6,TF_R_SP-TF_R_R6 // save r6, skip to sp
-	nop.i	0
-	;;
-} { .mmi
-	.mem.offset 0,0
-	st8.spill [r30]=r7,TF_R_R15-TF_R_R7 // save r7, skip to r15
-	st8	[r31]=r18		// save sp
-	nop.i	0
-	;; 
-} { .mmb
-	st8	[r30]=r15		// save r15 (syscall number)
-	add	sp=-(8*8),sp		// reserve stack for arguments
-	br.call.sptk.few b0=Lsaveargs	// dump args
-} { .mmb
-	mov.m	r13=ar.k4		// processor globals
-	nop.m	0
-	bsw.1				// switch back to bank 1
-	;;
-} { .mmb
-	mov	r16=sp			// point at args
-	mov.m	r17=ar.k5		// for calculating ndirty
-	cover				// preserve user register frame
-	;;
-} { .mmi
-	mov	r18=cr.ifs		// record user's CFM
-	mov.m	r19=ar.bsp		// ndirty = ar.bsp - kbsp
-	add	sp=-16,sp		// reserve scratch space
-	;;
-} { .mmi
-	add	r20=TF_CR_IFS+(8*8),r16	// point at cr.ifs
-	ssm	psr.ic|psr.dt		// reenable traps and translation
-	sub	r19=r19,r17		// calculate ndirty
-	;;
-} { .mmi
-	srlz.i				// serialize psr.ic and psr.dt
-	;;
-	ssm	psr.i			// safe to take interrupts again
-	add	r21=TF_NDIRTY+(8*8),r16	// point at ndirty
-	;;
-} { .mmi
-	st8	[r20]=r18		// save cr.ifs
-	st8	[r21]=r19		// save ndirty
-	;;
-} { .mmi
-	alloc	r14=ar.pfs,0,1,3,0
-	srlz.d				// serialize psr.i
-	add	loc0=(8*8),r16		// remember where trapframe is
-	;;
-} { .mlx
-	mov	out0=r15		// syscall number (from user)
-	movl	gp=__gp			// kernel globals
-} { .mmb
-	mov	out1=r16		// arguments
-	add	out2=(8*8),r16		// trapframe pointer
-	br.call.sptk.many rp=syscall	// do the work
-} { .mmi
-3:	rsm	psr.i			// we know that psr.i == 1
-	add	r14=PC_CURTHREAD,r13	// &curthread
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r14=[r14]		// curthread
-	;;
-	nop.i	0
-	;;
-} { .mmi
-	;;
-	add	r14=TD_FLAGS,r14	// &curthread->td_flags
-	nop.i	0
-	;;
-} { .mmi
-	ld4	r14=[r14]		// curkse->ke_flags
-	;;
-	nop.m	0
-	tbit.nz	p6,p7=r14,11		// TDF_ASTPENDING
-	;;
-} { .mib
-	nop.m	0
-(p7)	tbit.nz.or.andcm p6,p7=r14,16	// TDF_NEEDRESCHED
-(p7)	br.cond.dptk 2f
-	;;
-} { .mmi
-	ssm	psr.i			// restore interrupts
-	;;
-	srlz.d
-	mov	out0=loc0		// trapframe argument to ast()
-} { .mib
-	nop.m	0
-	nop.i	0
-	br.call.sptk.many rp=ast
-} { .mib
-	nop.m	0
-	nop.i	0
-	br	3b
-} { .mii
-2:	ld8	r14=[loc0]		// check tf_flags
-	dep	r15=0,loc0,61,3		// physical address of trapframe
-	;;
-	tbit.z p6,p0=r14,0		// check FRAME_SYSCALL bit
-	;;
-} { .mib
-(p6)	add	sp=-16,loc0		// do a full restore if clear
-	add	r16=SIZEOF_TRAPFRAME,loc0 // new kernel sp
-(p6)	br.dpnt.many exception_restore
-} { .mmi
-	rsm 	psr.dt|psr.ic|psr.i	// get ready to restore
-	;;
-	srlz.i				// serialise psr.dt and psr.ic
-	add	r30=TF_R_R15,r15	// point at r15
-	;; 
-} { .mmi
-	alloc	r14=ar.pfs,0,0,0,0	// discard register frame
-	mov	ar.k6=r16		// restore kernel sp
-	add	r31=TF_R_SP,r15		// point at sp
-	;;
-} { .mmi
-	ld8	r15=[r30],TF_R_R10-TF_R_R15 // restore r15, skip to r10
-	ld8	sp=[r31],TF_R_R9-TF_R_SP // restore user sp, skip to r9
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r10=[r30],-16		// restore r10, skip to r8
-	ld8	r9=[r31],TF_R_R1-TF_R_R9 // restore r9, skip to r1
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r8=[r30],TF_B-TF_R_R8	// restore r8, skip to b0
-	ld8	r1=[r31],TF_AR_FPSR-TF_R_R1 // restore r1, skip to ar.fpsr
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r16=[r30],-32		// restore b0, skip to ar.ccv
-	ld8	r17=[r31],-16		// restore ar.fpsr, skip to ar.unat
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r18=[r30],-16		// restore ar.ccv, skip to ndirty
-	ld8	r19=[r31],-16		// restore ar.unat, skip to ar.rnat
-	mov	b0=r16
-	;;
-} { .mmi
-	ld8	r20=[r30],-16		// restore ndirty, skip to ar.bspstore
-	ld8	r21=[r31],-16		// restore ar.rnat, skip to cr.ifs
-	nop.i	0
-	;; 
-} { .mmi
-	ld8	r16=[r30],-16		// restore ar.bspstore, skip to ar.pfs
-	mov	ar.fpsr=r17
-	shl	r20=r20,16		// value for ar.rsc
-	;; 
-} { .mmi
-	ld8	r22=[r31],-16		// restore cr.ifs, skip to ar.rsc
-	mov.m	ar.ccv=r18
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r17=[r30],-16		// restore ar.pfs, skip to pr
-	mov.m	ar.unat=r19
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r18=[r31],-32		// restore ar.rsc, skip to cr.ipsr
-	mov.m	ar.rsc=r20		// setup for loadrs
-	nop.i	0
-	;;
-} { .mmi
-	loadrs				// restore user stacked registers
-	;; 
-	mov.m	ar.bspstore=r16		// back to user backing store
-	mov.i	ar.pfs=r17	
-	;;
-} { .mmi
-	mov.m	ar.rnat=r21
-	mov.m	ar.rsc=r18
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r16=[r30],-32		// restore pr, skip to cr.iip
-	ld8	r17=[r31]		// restore cr.ipsr
-	nop.i	0
-	;;
-} { .mmi
-	ld8	r18=[r30]		// restore cr.iip
-	mov	cr.ifs=r22
-	nop.i	0
-	;;
-} { .mmi
-	mov	cr.iip=r18
-	mov	cr.ipsr=r17
-	mov	pr=r16,0x1ffff
-	;;
-} { .bbb
-	rfi
-}	
-
-	// This is done as a function call to make sure that we only
-	// have output registers in the register frame. It also gives
-	// us a chance to use alloc to round up to 8 arguments for
-	// simplicity.
-	//
-	// We are still running in physical mode with psr.ic==0 because
-	// we haven't yet covered the user's register frame to get a
-	// value for cr.ifs
-Lsaveargs:
-{ .mii
-	alloc	r14=ar.pfs,0,0,8,0	// round up to 8 outputs
-	extr.u	r31=sp,0,61		// physical address
-	;;
-	add	r30=8,r31
-	;;
-} { .mmi
-	st8	[r31]=r32,16
-	st8	[r30]=r33,16
-	;;
-} { .mmi
-	st8	[r31]=r34,16
-	st8	[r30]=r35,16
-	;; 
-} { .mmi
-	st8	[r31]=r36,16
-	st8	[r30]=r37,16
-	;; 
-} { .mmb
-	st8	[r31]=r38
-	st8	[r30]=r39
-	br.ret.sptk.many b0
-}
-	.global do_syscall_end
-do_syscall_end:
-
-END(do_syscall)
diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c
index 158b388..302c441 100644
--- a/sys/ia64/ia64/genassym.c
+++ b/sys/ia64/ia64/genassym.c
@@ -37,8 +37,6 @@
  * $FreeBSD$
  */
 
-#include "opt_ia32.h"
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/assym.h>
@@ -65,87 +63,57 @@
 #include <net/if.h>
 #include <netinet/in.h>
 
-#ifdef IA32
-ASSYM(IA32, IA32);
-#endif
-
-ASSYM(PC_CURTHREAD,	offsetof(struct pcpu, pc_curthread));
-ASSYM(PC_IDLETHREAD,	offsetof(struct pcpu, pc_idlethread));
-ASSYM(PC_FPCURTHREAD,	offsetof(struct pcpu, pc_fpcurthread));
-ASSYM(PC_CURPCB,	offsetof(struct pcpu, pc_curpcb));
-ASSYM(PC_CPUID,		offsetof(struct pcpu, pc_cpuid));
-ASSYM(PC_CURRENT_PMAP,	offsetof(struct pcpu, pc_current_pmap));
-
-ASSYM(MTX_LOCK,		offsetof(struct mtx, mtx_lock));
-ASSYM(MTX_RECURSE,	offsetof(struct mtx, mtx_recurse));
-ASSYM(MTX_UNOWNED,	MTX_UNOWNED);
+ASSYM(DT_NULL,		DT_NULL);
+ASSYM(DT_RELA,		DT_RELA);
+ASSYM(DT_RELAENT,	DT_RELAENT);
+ASSYM(DT_RELASZ,	DT_RELASZ);
+ASSYM(DT_SYMTAB,	DT_SYMTAB);
+ASSYM(DT_SYMENT,	DT_SYMENT);
 
-ASSYM(TD_PROC,		offsetof(struct thread, td_proc));
-ASSYM(TD_PCB,		offsetof(struct thread, td_pcb));
-ASSYM(TD_KSTACK,	offsetof(struct thread, td_kstack));
-ASSYM(TD_MD_FLAGS,	offsetof(struct thread, td_md.md_flags));
+ASSYM(EFAULT,		EFAULT);
+ASSYM(ENAMETOOLONG,	ENAMETOOLONG);
+ASSYM(ERESTART,		ERESTART);
 
-ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
+ASSYM(FRAME_SYSCALL,	FRAME_SYSCALL);
 
-ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
-ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
+ASSYM(KSTACK_PAGES,	KSTACK_PAGES);
 
-ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
+ASSYM(MC_PRESERVED,	offsetof(mcontext_t, mc_preserved));
+ASSYM(MC_PRESERVED_FP,	offsetof(mcontext_t, mc_preserved_fp));
+ASSYM(MC_SPECIAL,	offsetof(mcontext_t, mc_special));
+ASSYM(MC_SPECIAL_BSPSTORE, offsetof(mcontext_t, mc_special.bspstore));
+ASSYM(MC_SPECIAL_RNAT,	offsetof(mcontext_t, mc_special.rnat));
 
-ASSYM(FRAME_SYSCALL,	FRAME_SYSCALL);
+ASSYM(PAGE_SHIFT,	PAGE_SHIFT);
+ASSYM(PAGE_SIZE,	PAGE_SIZE);
 
-ASSYM(TF_CR_IPSR,	offsetof(struct trapframe, tf_cr_ipsr));
-ASSYM(TF_CR_IFS,	offsetof(struct trapframe, tf_cr_ifs));
-ASSYM(TF_NDIRTY,	offsetof(struct trapframe, tf_ndirty));
-ASSYM(TF_AR_FPSR,	offsetof(struct trapframe, tf_ar_fpsr));
-ASSYM(TF_B,		offsetof(struct trapframe, tf_b));
-ASSYM(TF_R,		offsetof(struct trapframe, tf_r));
-ASSYM(TF_R_R1,		offsetof(struct trapframe, tf_r[FRAME_R1]));
-ASSYM(TF_R_R2,		offsetof(struct trapframe, tf_r[FRAME_R2]));
-ASSYM(TF_R_R3,		offsetof(struct trapframe, tf_r[FRAME_R3]));
-ASSYM(TF_R_R4,		offsetof(struct trapframe, tf_r[FRAME_R4]));
-ASSYM(TF_R_R5,		offsetof(struct trapframe, tf_r[FRAME_R5]));
-ASSYM(TF_R_R6,		offsetof(struct trapframe, tf_r[FRAME_R6]));
-ASSYM(TF_R_R7,		offsetof(struct trapframe, tf_r[FRAME_R7]));
-ASSYM(TF_R_R8,		offsetof(struct trapframe, tf_r[FRAME_R8]));
-ASSYM(TF_R_R9,		offsetof(struct trapframe, tf_r[FRAME_R9]));
-ASSYM(TF_R_R10,		offsetof(struct trapframe, tf_r[FRAME_R10]));
-ASSYM(TF_R_R11,		offsetof(struct trapframe, tf_r[FRAME_R11]));
-ASSYM(TF_R_SP,		offsetof(struct trapframe, tf_r[FRAME_SP]));
-ASSYM(TF_R_R13,		offsetof(struct trapframe, tf_r[FRAME_R13]));
-ASSYM(TF_R_R14,		offsetof(struct trapframe, tf_r[FRAME_R14]));
-ASSYM(TF_R_R15,		offsetof(struct trapframe, tf_r[FRAME_R15]));
-ASSYM(TF_F,		offsetof(struct trapframe, tf_f));
+ASSYM(PC_CPUID,		offsetof(struct pcpu, pc_cpuid));
+ASSYM(PC_CURRENT_PMAP,	offsetof(struct pcpu, pc_current_pmap));
+ASSYM(PC_CURTHREAD,	offsetof(struct pcpu, pc_curthread));
+ASSYM(PC_IDLETHREAD,	offsetof(struct pcpu, pc_idlethread));
 
 ASSYM(PCB_CURRENT_PMAP,	offsetof(struct pcb, pcb_current_pmap));
 ASSYM(PCB_ONFAULT,	offsetof(struct pcb, pcb_onfault));
-ASSYM(PCB_RP,		offsetof(struct pcb, pcb_rp));
-ASSYM(PCB_UNAT47,	offsetof(struct pcb, pcb_unat47));
+ASSYM(PCB_SPECIAL_RP,	offsetof(struct pcb, pcb_special.rp));
 
-ASSYM(UC_MCONTEXT_MC_AR_BSP,  offsetof(ucontext_t, uc_mcontext.mc_ar_bsp));
-ASSYM(UC_MCONTEXT_MC_AR_RNAT, offsetof(ucontext_t, uc_mcontext.mc_ar_rnat));
+ASSYM(R_IA64_DIR64LSB,	R_IA64_DIR64LSB);
+ASSYM(R_IA64_FPTR64LSB,	R_IA64_FPTR64LSB);
+ASSYM(R_IA64_NONE,	R_IA64_NONE);
+ASSYM(R_IA64_REL64LSB,	R_IA64_REL64LSB);
 
-ASSYM(EFAULT,		EFAULT);
-ASSYM(ENAMETOOLONG,	ENAMETOOLONG);
+ASSYM(SIZEOF_PCB,	sizeof(struct pcb));
+ASSYM(SIZEOF_SPECIAL,	sizeof(struct _special));
+ASSYM(SIZEOF_TRAPFRAME,	sizeof(struct trapframe));
 
-ASSYM(PAGE_SHIFT,	PAGE_SHIFT); 
-ASSYM(PAGE_SIZE,	PAGE_SIZE);
-ASSYM(KSTACK_PAGES,	KSTACK_PAGES);
+ASSYM(TD_FLAGS,		offsetof(struct thread, td_flags));
+ASSYM(TD_KSTACK,	offsetof(struct thread, td_kstack));
+ASSYM(TD_PCB,		offsetof(struct thread, td_pcb));
 
-ASSYM(SIZEOF_TRAPFRAME,	sizeof(struct trapframe));
-ASSYM(SIZEOF_PCB,	sizeof(struct pcb));
+ASSYM(TDF_ASTPENDING,	TDF_ASTPENDING);
+ASSYM(TDF_NEEDRESCHED,	TDF_NEEDRESCHED);
 
-ASSYM(DT_NULL,		DT_NULL);
-ASSYM(DT_RELA,		DT_RELA);
-ASSYM(DT_RELASZ,	DT_RELASZ);
-ASSYM(DT_SYMTAB,	DT_SYMTAB);
-ASSYM(DT_SYMENT,	DT_SYMENT);
-ASSYM(DT_RELAENT,	DT_RELAENT);
-ASSYM(R_IA64_NONE,	R_IA64_NONE);
-ASSYM(R_IA64_DIR64LSB,	R_IA64_DIR64LSB);
-ASSYM(R_IA64_FPTR64LSB,	R_IA64_FPTR64LSB);
-ASSYM(R_IA64_REL64LSB,	R_IA64_REL64LSB);
+ASSYM(TF_SPECIAL_NDIRTY, offsetof(struct trapframe, tf_special.ndirty));
+
+ASSYM(UC_MCONTEXT,	offsetof(ucontext_t, uc_mcontext));
 
-ASSYM(PAL_PTCE_INFO,	PAL_PTCE_INFO);
-ASSYM(PAL_FREQ_RATIOS,	PAL_FREQ_RATIOS);
-ASSYM(PAL_VM_SUMMARY,	PAL_VM_SUMMARY);
+ASSYM(VM_MAX_ADDRESS,	VM_MAX_ADDRESS);
diff --git a/sys/ia64/ia64/interrupt.c b/sys/ia64/ia64/interrupt.c
index bc2d324..a9539a9 100644
--- a/sys/ia64/ia64/interrupt.c
+++ b/sys/ia64/ia64/interrupt.c
@@ -152,6 +152,9 @@ interrupt(u_int64_t vector, struct trapframe *framep)
 	} else if (vector == ipi_vector[IPI_AST]) {
 		asts[PCPU_GET(cpuid)]++;
 		CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
+	} else if (vector == ipi_vector[IPI_HIGH_FP]) {
+		if (PCPU_GET(fpcurthread) != NULL)
+			ia64_highfp_save(PCPU_GET(fpcurthread));
 	} else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
 		rdvs[PCPU_GET(cpuid)]++;
 		CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
diff --git a/sys/ia64/ia64/locore.S b/sys/ia64/ia64/locore.S
index 61671b6..69550d1 100644
--- a/sys/ia64/ia64/locore.S
+++ b/sys/ia64/ia64/locore.S
@@ -25,32 +25,6 @@
  *
  * $FreeBSD$
  */
-/*
- * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- *
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- *
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- *
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- */
 
 #include <machine/asm.h>
 #include <machine/ia64_cpu.h>
@@ -77,22 +51,33 @@ kstack:	.space KSTACK_PAGES * PAGE_SIZE
  * register r8.
  */
 ENTRY(__start, 1)
+{	.mlx
 	mov	ar.rsc=0
 	movl	r16=ia64_vector_table	// set up IVT early
 	;;
+}
+{	.mlx
 	mov	cr.iva=r16
 	movl	r16=kstack
 	;;
+}
+{	.mmi
 	srlz.i
 	;;
+	ssm	IA64_PSR_DFH
 	mov	r17=KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16
 	;;
+}
+{	.mlx
 	add	sp=r16,r17		// proc0's stack
 	movl	gp=__gp			// find kernel globals
 	;;
+}
+{	.mlx
 	mov	ar.bspstore=r16		// switch backing store
 	movl	r16=pa_bootinfo
 	;;
+}
 	st8	[r16]=r8		// save the PA of the bootinfo block
 	loadrs				// invalidate regs
 	;;
@@ -117,7 +102,7 @@ ENTRY(__start, 1)
 	;;
 	ld8	out0=[out0]
 	;; 
-	add	r16=PCB_RP,out0		// return to mi_startup_trampoline
+	add	r16=PCB_SPECIAL_RP,out0	// return to mi_startup_trampoline
 	movl	r17=mi_startup_trampoline
 	;;
 	st8	[r16]=r17
@@ -133,7 +118,7 @@ ENTRY(mi_startup_trampoline, 0)
 	.prologue
 	.save	rp,r0
 	.body
-	
+
 	br.call.sptk.many rp=mi_startup
 
 	// Should never happen
@@ -141,6 +126,59 @@ ENTRY(mi_startup_trampoline, 0)
 
 END(mi_startup_trampoline)
 
+/*
+ * fork_trampoline()
+ *
+ * Arrange for a function to be invoked neatly, after a cpu_switch().
+ *
+ * Invokes fork_exit() passing in three arguments: a callout function, an
+ * argument to the callout, and a trapframe pointer.  For child processes
+ * returning from fork(2), the argument is a pointer to the child process.
+ *
+ * The callout function and its argument is in the trapframe in scratch
+ * registers r2 and r3.
+ */
+ENTRY(fork_trampoline, 0)
+	.prologue
+	.save	rp,r0
+	.body
+{	.mmi
+	alloc		r14=ar.pfs,0,0,3,0
+	add		r15=32+SIZEOF_SPECIAL+8,sp
+	add		r16=32+SIZEOF_SPECIAL+16,sp
+	;;
+}
+{	.mmi
+	ld8		out0=[r15]
+	ld8		out1=[r16]
+	nop		0
+}
+{	.mfb
+	add		out2=16,sp
+	nop		0
+	br.call.sptk	rp=fork_exit
+	;;
+}
+	// If we get back here, it means we're a user space process that's
+	// the immediate result of fork(2).
+	.global		enter_userland
+	.type		enter_userland, @function
+enter_userland:
+{	.mmi
+	add		r14=24,sp
+	;;
+	ld8		r14=[r14]
+	nop		0
+	;;
+}
+{	.mbb
+	cmp.eq		p6,p7=r0,r14
+(p6)	br.sptk		exception_restore
+(p7)	br.sptk		epc_syscall_return
+	;;
+}
+END(fork_trampoline)
+
 #ifdef SMP
 /*
  * AP wake-up entry point. The handoff state is similar as for the BSP,
@@ -198,7 +236,7 @@ ENTRY(os_boot_rendez,0)
 	;;
 1:	mov	r16 = ip
 	add	r17 = 2f-1b, r17
-	movl	r18 = (IA64_PSR_AC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT|IA64_PSR_BN)
+	movl	r18 = (IA64_PSR_AC|IA64_PSR_BN|IA64_PSR_DFH|IA64_PSR_DT|IA64_PSR_IC|IA64_PSR_IT|IA64_PSR_RT)
 	;;
 	add	r17 = r17, r16
 	mov	cr.ipsr = r18
@@ -209,142 +247,54 @@ ENTRY(os_boot_rendez,0)
 	rfi
 
 	.align	32
-2:	movl	r16 = ia64_vector_table			// set up IVT early
+2:
+{	.mlx
+	mov	ar.rsc = 0
+	movl	r16 = ia64_vector_table			// set up IVT early
 	;;
+}
+{	.mlx
 	mov	cr.iva = r16
+	movl	r16 = ap_stack
 	;;
+}
+{	.mmi
 	srlz.i
 	;;
-	movl	r16 = ap_stack
-	movl	r17 = ap_pcpu
-	mov	ar.rsc = 0
-	movl	gp = __gp
-	;;
 	ld8	r16 = [r16]
-	ld8	r17 = [r17]
 	mov	r18 = KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16
 	;;
-	add	sp = r18, r16
+}
+{	.mlx
 	mov	ar.bspstore = r16
-	mov	ar.k4 = r17
-	mov	r13 = r17	/* gas doesn't know tp as an alias for r13 */
+	movl	gp = __gp
 	;;
+}
+{	.mmi
 	loadrs
-	movl	r16 = ia64_pal_base
-	;;
-	mov	ar.rsc = 3
-	ld8	r16 = [r16]
-	;;
-	cmp.eq	p1, p0 = 0, r16
-(p1)	br.cond.spnt	1f
-	;;
-	mov	r18 = 28<<2
-	movl	r17 = 7<<61
-	;;
-	mov	cr.itir = r18
-	or	r17 = r17, r16
-	mov	r16 = (PTE_P|PTE_MA_WB|PTE_A|PTE_D|PTE_PL_KERN|PTE_AR_RWX)
-	;;
-	mov	cr.ifa = r17
-	extr.u  r18 = r17, 12, 38
-	;;
-	srlz.i
-	shl	r18 = r18, 12
-	;;
-	add	r17 = 1, r0
-	or	r16 = r16, r18
-	;;
-	itr.i	itr[r17] = r16
 	;;
-	srlz.i
-	;;
-1:	alloc	r16 = ar.pfs, 0, 0, 0, 0
+	alloc	r17 = ar.pfs, 0, 0, 0, 0
+	add	sp = r18, r16
 	;;
+}
+{	.mfb
+	mov	ar.rsc = 3
+	nop	0
 	br.call.sptk.few rp = ia64_ap_startup
+	;;
+}
 	/* NOT REACHED */
-9:	br	9b
+9:
+{	.mfb
+	nop	0
+	nop	0
+	br.sptk	9b
+	;;
+}
 END(os_boot_rendez)
 
 #endif /* !SMP */
 
-/**************************************************************************/
-
-/*
- * Signal "trampoline" code. Invoked from RTE setup by sendsig().
- *
- * On entry, registers look like:
- *
- *      r14	signal number
- *      r15	pointer to siginfo_t
- *	r16	pointer to signal context frame (scp)
- *      r17	address of handler function descriptor
- *	r18	address of new backing store (if any)
- *      sp+16	pointer to sigframe
- */
-
-ENTRY(sigcode,0)
-	ld8	r8=[r17],8		// function address
-	;;
-	ld8	gp=[r17]		// function's gp value
-	mov	b6=r8			// transfer to a branch register
-	cover
-	;;
-	add	r8=UC_MCONTEXT_MC_AR_BSP,r16 // address or mc_ar_bsp
-	mov	r9=ar.bsp		// save ar.bsp
-	;;
-	st8	[r8]=r9
-	cmp.eq	p1,p2=r0,r18		// check for new bs
-(p1)	br.cond.sptk.few 1f		// branch if not switching
-	flushrs				// flush out to old bs
-	mov	ar.rsc=0		// switch off RSE
-	add	r8=UC_MCONTEXT_MC_AR_RNAT,r16 // address of mc_ar_rnat
-	;;
-	mov	r9=ar.rnat		// value of ar.rnat after flush
-	mov	ar.bspstore=r18		// point at new bs
-	;;
-	st8	[r8]=r9			// remember ar.rnat
-	mov	ar.rsc=15		// XXX bogus value - check
-	invala
-	;; 
-1:	alloc	r5=ar.pfs,0,0,3,0	// register frame for call
-	;;
-	mov	out0=r14		// signal number
-	mov	out1=r15		// siginfo
-	mov	out2=r16		// ucontext
-	mov	r4=r16			// save from call
-	br.call.sptk.few rp=b6		// call the signal handler
-	;; 
-	alloc	r14=ar.pfs,0,0,0,0	// discard call frame
-	;; 
-	flushrs
-	;; 
-(p1)	br.cond.sptk.few 2f		// note: p1 is preserved
-	mov	ar.rsc=0
-	add	r8=UC_MCONTEXT_MC_AR_RNAT,r4 // address of mc_ar_rnat
-	;;
-	ld8	r9=[r8]
-	;; 
-	add	r8=UC_MCONTEXT_MC_AR_BSP,r4 // address of mc_ar_bsp
-	;;
-	ld8	r10=[r8]
-	;;
-	mov	ar.bspstore=r10
-	;;
-	mov	ar.rnat=r9
-	mov	ar.rsc=15
-	;; 
-2:	CALLSYS_NOERROR(sigreturn)	// call sigreturn()
-	alloc	r14=ar.pfs,0,0,1,0 ;;
-	mov	out0=ret0		// if that failed, get error code
-	CALLSYS_NOERROR(exit)		// and call exit() with it.
-XENTRY(esigcode)
-	END(sigcode)
-
-	.data
-	EXPORT(szsigcode)
-	.quad	esigcode-sigcode
-	.text
-
 /*
  * Create a default interrupt name table. The first entry (vector 0) is
  * hardwaired to the clock interrupt.
diff --git a/sys/ia64/ia64/locore.s b/sys/ia64/ia64/locore.s
index 61671b6..69550d1 100644
--- a/sys/ia64/ia64/locore.s
+++ b/sys/ia64/ia64/locore.s
@@ -25,32 +25,6 @@
  *
  * $FreeBSD$
  */
-/*
- * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
- * All rights reserved.
- *
- * Author: Chris G. Demetriou
- *
- * Permission to use, copy, modify and distribute this software and
- * its documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- *
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
- * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- *
- * Carnegie Mellon requests users of this software to return to
- *
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- *
- * any improvements or extensions that they make and grant Carnegie the
- * rights to redistribute these changes.
- */
 
 #include <machine/asm.h>
 #include <machine/ia64_cpu.h>
@@ -77,22 +51,33 @@ kstack:	.space KSTACK_PAGES * PAGE_SIZE
  * register r8.
  */
 ENTRY(__start, 1)
+{	.mlx
 	mov	ar.rsc=0
 	movl	r16=ia64_vector_table	// set up IVT early
 	;;
+}
+{	.mlx
 	mov	cr.iva=r16
 	movl	r16=kstack
 	;;
+}
+{	.mmi
 	srlz.i
 	;;
+	ssm	IA64_PSR_DFH
 	mov	r17=KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16
 	;;
+}
+{	.mlx
 	add	sp=r16,r17		// proc0's stack
 	movl	gp=__gp			// find kernel globals
 	;;
+}
+{	.mlx
 	mov	ar.bspstore=r16		// switch backing store
 	movl	r16=pa_bootinfo
 	;;
+}
 	st8	[r16]=r8		// save the PA of the bootinfo block
 	loadrs				// invalidate regs
 	;;
@@ -117,7 +102,7 @@ ENTRY(__start, 1)
 	;;
 	ld8	out0=[out0]
 	;; 
-	add	r16=PCB_RP,out0		// return to mi_startup_trampoline
+	add	r16=PCB_SPECIAL_RP,out0	// return to mi_startup_trampoline
 	movl	r17=mi_startup_trampoline
 	;;
 	st8	[r16]=r17
@@ -133,7 +118,7 @@ ENTRY(mi_startup_trampoline, 0)
 	.prologue
 	.save	rp,r0
 	.body
-	
+
 	br.call.sptk.many rp=mi_startup
 
 	// Should never happen
@@ -141,6 +126,59 @@ ENTRY(mi_startup_trampoline, 0)
 
 END(mi_startup_trampoline)
 
+/*
+ * fork_trampoline()
+ *
+ * Arrange for a function to be invoked neatly, after a cpu_switch().
+ *
+ * Invokes fork_exit() passing in three arguments: a callout function, an
+ * argument to the callout, and a trapframe pointer.  For child processes
+ * returning from fork(2), the argument is a pointer to the child process.
+ *
+ * The callout function and its argument is in the trapframe in scratch
+ * registers r2 and r3.
+ */
+ENTRY(fork_trampoline, 0)
+	.prologue
+	.save	rp,r0
+	.body
+{	.mmi
+	alloc		r14=ar.pfs,0,0,3,0
+	add		r15=32+SIZEOF_SPECIAL+8,sp
+	add		r16=32+SIZEOF_SPECIAL+16,sp
+	;;
+}
+{	.mmi
+	ld8		out0=[r15]
+	ld8		out1=[r16]
+	nop		0
+}
+{	.mfb
+	add		out2=16,sp
+	nop		0
+	br.call.sptk	rp=fork_exit
+	;;
+}
+	// If we get back here, it means we're a user space process that's
+	// the immediate result of fork(2).
+	.global		enter_userland
+	.type		enter_userland, @function
+enter_userland:
+{	.mmi
+	add		r14=24,sp
+	;;
+	ld8		r14=[r14]
+	nop		0
+	;;
+}
+{	.mbb
+	cmp.eq		p6,p7=r0,r14
+(p6)	br.sptk		exception_restore
+(p7)	br.sptk		epc_syscall_return
+	;;
+}
+END(fork_trampoline)
+
 #ifdef SMP
 /*
  * AP wake-up entry point. The handoff state is similar as for the BSP,
@@ -198,7 +236,7 @@ ENTRY(os_boot_rendez,0)
 	;;
 1:	mov	r16 = ip
 	add	r17 = 2f-1b, r17
-	movl	r18 = (IA64_PSR_AC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT|IA64_PSR_BN)
+	movl	r18 = (IA64_PSR_AC|IA64_PSR_BN|IA64_PSR_DFH|IA64_PSR_DT|IA64_PSR_IC|IA64_PSR_IT|IA64_PSR_RT)
 	;;
 	add	r17 = r17, r16
 	mov	cr.ipsr = r18
@@ -209,142 +247,54 @@ ENTRY(os_boot_rendez,0)
 	rfi
 
 	.align	32
-2:	movl	r16 = ia64_vector_table			// set up IVT early
+2:
+{	.mlx
+	mov	ar.rsc = 0
+	movl	r16 = ia64_vector_table			// set up IVT early
 	;;
+}
+{	.mlx
 	mov	cr.iva = r16
+	movl	r16 = ap_stack
 	;;
+}
+{	.mmi
 	srlz.i
 	;;
-	movl	r16 = ap_stack
-	movl	r17 = ap_pcpu
-	mov	ar.rsc = 0
-	movl	gp = __gp
-	;;
 	ld8	r16 = [r16]
-	ld8	r17 = [r17]
 	mov	r18 = KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16
 	;;
-	add	sp = r18, r16
+}
+{	.mlx
 	mov	ar.bspstore = r16
-	mov	ar.k4 = r17
-	mov	r13 = r17	/* gas doesn't know tp as an alias for r13 */
+	movl	gp = __gp
 	;;
+}
+{	.mmi
 	loadrs
-	movl	r16 = ia64_pal_base
-	;;
-	mov	ar.rsc = 3
-	ld8	r16 = [r16]
-	;;
-	cmp.eq	p1, p0 = 0, r16
-(p1)	br.cond.spnt	1f
-	;;
-	mov	r18 = 28<<2
-	movl	r17 = 7<<61
-	;;
-	mov	cr.itir = r18
-	or	r17 = r17, r16
-	mov	r16 = (PTE_P|PTE_MA_WB|PTE_A|PTE_D|PTE_PL_KERN|PTE_AR_RWX)
-	;;
-	mov	cr.ifa = r17
-	extr.u  r18 = r17, 12, 38
-	;;
-	srlz.i
-	shl	r18 = r18, 12
-	;;
-	add	r17 = 1, r0
-	or	r16 = r16, r18
-	;;
-	itr.i	itr[r17] = r16
 	;;
-	srlz.i
-	;;
-1:	alloc	r16 = ar.pfs, 0, 0, 0, 0
+	alloc	r17 = ar.pfs, 0, 0, 0, 0
+	add	sp = r18, r16
 	;;
+}
+{	.mfb
+	mov	ar.rsc = 3
+	nop	0
 	br.call.sptk.few rp = ia64_ap_startup
+	;;
+}
 	/* NOT REACHED */
-9:	br	9b
+9:
+{	.mfb
+	nop	0
+	nop	0
+	br.sptk	9b
+	;;
+}
 END(os_boot_rendez)
 
 #endif /* !SMP */
 
-/**************************************************************************/
-
-/*
- * Signal "trampoline" code. Invoked from RTE setup by sendsig().
- *
- * On entry, registers look like:
- *
- *      r14	signal number
- *      r15	pointer to siginfo_t
- *	r16	pointer to signal context frame (scp)
- *      r17	address of handler function descriptor
- *	r18	address of new backing store (if any)
- *      sp+16	pointer to sigframe
- */
-
-ENTRY(sigcode,0)
-	ld8	r8=[r17],8		// function address
-	;;
-	ld8	gp=[r17]		// function's gp value
-	mov	b6=r8			// transfer to a branch register
-	cover
-	;;
-	add	r8=UC_MCONTEXT_MC_AR_BSP,r16 // address or mc_ar_bsp
-	mov	r9=ar.bsp		// save ar.bsp
-	;;
-	st8	[r8]=r9
-	cmp.eq	p1,p2=r0,r18		// check for new bs
-(p1)	br.cond.sptk.few 1f		// branch if not switching
-	flushrs				// flush out to old bs
-	mov	ar.rsc=0		// switch off RSE
-	add	r8=UC_MCONTEXT_MC_AR_RNAT,r16 // address of mc_ar_rnat
-	;;
-	mov	r9=ar.rnat		// value of ar.rnat after flush
-	mov	ar.bspstore=r18		// point at new bs
-	;;
-	st8	[r8]=r9			// remember ar.rnat
-	mov	ar.rsc=15		// XXX bogus value - check
-	invala
-	;; 
-1:	alloc	r5=ar.pfs,0,0,3,0	// register frame for call
-	;;
-	mov	out0=r14		// signal number
-	mov	out1=r15		// siginfo
-	mov	out2=r16		// ucontext
-	mov	r4=r16			// save from call
-	br.call.sptk.few rp=b6		// call the signal handler
-	;; 
-	alloc	r14=ar.pfs,0,0,0,0	// discard call frame
-	;; 
-	flushrs
-	;; 
-(p1)	br.cond.sptk.few 2f		// note: p1 is preserved
-	mov	ar.rsc=0
-	add	r8=UC_MCONTEXT_MC_AR_RNAT,r4 // address of mc_ar_rnat
-	;;
-	ld8	r9=[r8]
-	;; 
-	add	r8=UC_MCONTEXT_MC_AR_BSP,r4 // address of mc_ar_bsp
-	;;
-	ld8	r10=[r8]
-	;;
-	mov	ar.bspstore=r10
-	;;
-	mov	ar.rnat=r9
-	mov	ar.rsc=15
-	;; 
-2:	CALLSYS_NOERROR(sigreturn)	// call sigreturn()
-	alloc	r14=ar.pfs,0,0,1,0 ;;
-	mov	out0=ret0		// if that failed, get error code
-	CALLSYS_NOERROR(exit)		// and call exit() with it.
-XENTRY(esigcode)
-	END(sigcode)
-
-	.data
-	EXPORT(szsigcode)
-	.quad	esigcode-sigcode
-	.text
-
 /*
  * Create a default interrupt name table. The first entry (vector 0) is
  * hardwaired to the clock interrupt.
diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index dd3fc59..18d2083 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -73,6 +73,9 @@
 #include <machine/mca.h>
 #include <machine/pal.h>
 #include <machine/sal.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 #include <machine/bootinfo.h>
 #include <machine/mutex.h>
 #include <machine/vmparam.h>
@@ -83,7 +86,6 @@
 #include <machine/sigframe.h>
 #include <machine/efi.h>
 #include <machine/inst.h>
-#include <machine/rse.h>
 #include <machine/unwind.h>
 #include <i386/include/specialreg.h>
 
@@ -102,6 +104,10 @@ vm_offset_t proc0kstack;
 
 extern u_int64_t kernel_text[], _end[];
 
+extern u_int64_t ia64_gateway_page[];
+extern u_int64_t break_sigtramp[];
+extern u_int64_t epc_sigtramp[];
+
 FPSWA_INTERFACE *fpswa_interface;
 
 u_int64_t ia64_pal_base;
@@ -206,6 +212,44 @@ cpu_startup(dummy)
 }
 
 void
+cpu_switch(struct thread *old, struct thread *new)
+{
+	struct pcb *oldpcb, *newpcb;
+
+	oldpcb = old->td_pcb;
+	oldpcb->pcb_current_pmap = PCPU_GET(current_pmap);
+#if IA32
+	ia32_savectx(oldpcb);
+#endif
+	if (!savectx(oldpcb)) {
+		newpcb = new->td_pcb;
+		pmap_install(newpcb->pcb_current_pmap);
+		PCPU_SET(curthread, new);
+#if IA32
+		ia32_restorectx(newpcb);
+#endif
+		restorectx(newpcb);
+	}
+}
+
+void
+cpu_throw(struct thread *old __unused, struct thread *new)
+{
+	struct pcb *newpcb;
+
+	newpcb = new->td_pcb;
+	pmap_install(newpcb->pcb_current_pmap);
+	PCPU_SET(curthread, new);
+#if IA32
+	ia32_restorectx(newpcb);
+#endif
+	restorectx(newpcb);
+	/* We should not get here. */
+	panic("cpu_throw: restorectx() returned");
+	/* NOTREACHED */
+}
+
+void
 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 {
 	KASSERT(size >= sizeof(struct pcpu) + sizeof(struct pcb),
@@ -278,16 +322,20 @@ map_pal_code(void)
 	pte.pte_ar = PTE_AR_RWX;
 	pte.pte_ppn = ia64_pal_base >> 12;
 
-	__asm __volatile("mov %0=psr;;" : "=r" (psr));
-	__asm __volatile("rsm psr.ic|psr.i;; srlz.i;;");
-	__asm __volatile("mov cr.ifa=%0" ::
+	__asm __volatile("ptr.d %0,%1; ptr.i %0,%1" ::
+	    "r"(IA64_PHYS_TO_RR7(ia64_pal_base)), "r"(28 << 2));
+
+	__asm __volatile("mov	%0=psr" : "=r"(psr));
+	__asm __volatile("rsm	psr.ic|psr.i");
+	__asm __volatile("srlz.i");
+	__asm __volatile("mov	cr.ifa=%0" ::
 	    "r"(IA64_PHYS_TO_RR7(ia64_pal_base)));
-	__asm __volatile("mov cr.itir=%0" :: "r"(28 << 2));
-	__asm __volatile("srlz.i;;");
-	__asm __volatile("itr.i itr[%0]=%1;;" ::
-	    "r"(2), "r"(*(u_int64_t*)&pte));
-	__asm __volatile("srlz.i;;");
-	__asm __volatile("mov psr.l=%0;; srlz.i;;" :: "r" (psr));
+	__asm __volatile("mov	cr.itir=%0" :: "r"(28 << 2));
+	__asm __volatile("itr.d	dtr[%0]=%1" :: "r"(1), "r"(*(u_int64_t*)&pte));
+	__asm __volatile("srlz.d");		/* XXX not needed. */
+	__asm __volatile("itr.i	itr[%0]=%1" :: "r"(1), "r"(*(u_int64_t*)&pte));
+	__asm __volatile("mov	psr.l=%0" :: "r" (psr));
+	__asm __volatile("srlz.i");
 }
 
 void
@@ -306,20 +354,53 @@ map_port_space(void)
 	pte.pte_a = 1;
 	pte.pte_d = 1;
 	pte.pte_pl = PTE_PL_KERN;
-	pte.pte_ar = PTE_AR_RWX;
+	pte.pte_ar = PTE_AR_RW;
 	pte.pte_ppn = ia64_port_base >> 12;
 
-	__asm __volatile("mov %0=psr;;" : "=r" (psr));
-	__asm __volatile("rsm psr.ic|psr.i;; srlz.i;;");
-	__asm __volatile("mov cr.ifa=%0" ::
-	    "r"(IA64_PHYS_TO_RR6(ia64_port_base)));
+	__asm __volatile("ptr.d %0,%1" :: "r"(ia64_port_base), "r"(24 << 2));
+
+	__asm __volatile("mov	%0=psr" : "=r" (psr));
+	__asm __volatile("rsm	psr.ic|psr.i");
+	__asm __volatile("srlz.d");
+	__asm __volatile("mov	cr.ifa=%0" :: "r"(ia64_port_base));
 	/* XXX We should use the size from the memory descriptor. */
-	__asm __volatile("mov cr.itir=%0" :: "r"(24 << 2));
-	__asm __volatile("srlz.i;;");
-	__asm __volatile("itr.i itr[%0]=%1;;" ::
-	    "r"(1), "r"(*(u_int64_t*)&pte));
-	__asm __volatile("srlz.i;;");
-	__asm __volatile("mov psr.l=%0;; srlz.i;;" :: "r" (psr));
+	__asm __volatile("mov	cr.itir=%0" :: "r"(24 << 2));
+	__asm __volatile("itr.d dtr[%0]=%1" :: "r"(2), "r"(*(u_int64_t*)&pte));
+	__asm __volatile("mov	psr.l=%0" :: "r" (psr));
+	__asm __volatile("srlz.d");
+}
+
+void
+map_gateway_page(void)
+{
+	struct ia64_pte pte;
+	u_int64_t psr;
+
+	bzero(&pte, sizeof(pte));
+	pte.pte_p = 1;
+	pte.pte_ma = PTE_MA_WB;
+	pte.pte_a = 1;
+	pte.pte_d = 1;
+	pte.pte_pl = PTE_PL_KERN;
+	pte.pte_ar = PTE_AR_X_RX;
+	pte.pte_ppn = IA64_RR_MASK((u_int64_t)ia64_gateway_page) >> 12;
+
+	__asm __volatile("ptr.d %0,%1; ptr.i %0,%1" ::
+	    "r"(VM_MAX_ADDRESS), "r"(PAGE_SHIFT << 2));
+
+	__asm __volatile("mov	%0=psr" : "=r"(psr));
+	__asm __volatile("rsm	psr.ic|psr.i");
+	__asm __volatile("srlz.i");
+	__asm __volatile("mov	cr.ifa=%0" :: "r"(VM_MAX_ADDRESS));
+	__asm __volatile("mov	cr.itir=%0" :: "r"(PAGE_SHIFT << 2));
+	__asm __volatile("itr.d	dtr[%0]=%1" :: "r"(3), "r"(*(u_int64_t*)&pte));
+	__asm __volatile("srlz.d");		/* XXX not needed. */
+	__asm __volatile("itr.i	itr[%0]=%1" :: "r"(3), "r"(*(u_int64_t*)&pte));
+	__asm __volatile("mov	psr.l=%0" :: "r" (psr));
+	__asm __volatile("srlz.i");
+
+	/* Expose the mapping to userland in ar.k5 */
+	ia64_set_k5(VM_MAX_ADDRESS);
 }
 
 static void
@@ -639,21 +720,12 @@ ia64_init(u_int64_t arg1, u_int64_t arg2)
 	/*
 	 * Setup the global data for the bootstrap cpu.
 	 */
-	pcpup = (struct pcpu *) pmap_steal_memory(PAGE_SIZE);
+	pcpup = (struct pcpu *)pmap_steal_memory(PAGE_SIZE);
+	ia64_set_k4((u_int64_t)pcpup);
 	pcpu_init(pcpup, 0, PAGE_SIZE);
-	ia64_set_k4((u_int64_t) pcpup);
 	PCPU_SET(curthread, &thread0);
 
 	/*
-	 * Set ia32 control registers.
-	 */
-	ia64_set_cflg((CR0_PE | CR0_PG)
-		      | ((long)(CR4_XMM | CR4_FXSR) << 32));
-
-	/* We pretend to own FP state so that ia64_fpstate_check() works */
-	PCPU_SET(fpcurthread, &thread0);
-
-	/*
 	 * Initialize the rest of proc 0's PCB.
 	 *
 	 * Set the kernel sp, reserving space for an (empty) trapframe,
@@ -663,8 +735,11 @@ ia64_init(u_int64_t arg1, u_int64_t arg2)
 	 * XXX what is all this +/- 16 stuff?
 	 */
 	thread0.td_frame = (struct trapframe *)thread0.td_pcb - 1;
-	thread0.td_pcb->pcb_sp = (u_int64_t)thread0.td_frame - 16;
-	thread0.td_pcb->pcb_ar_bsp = (u_int64_t)proc0kstack;
+	thread0.td_frame->tf_length = sizeof(struct trapframe);
+	thread0.td_frame->tf_flags = FRAME_SYSCALL;
+	thread0.td_pcb->pcb_special.sp =
+	    (u_int64_t)thread0.td_frame - 16;
+	thread0.td_pcb->pcb_special.bspstore = (u_int64_t)proc0kstack;
 
 	mutex_init();
 
@@ -732,40 +807,29 @@ DELAY(int n)
 }
 
 /*
- * Send an interrupt to process.
- *
- * Stack is set up to allow sigcode stored
- * at top to call routine, followed by kcall
- * to sigreturn routine below.  After sigreturn
- * resets the signal mask, the stack, and the
- * frame pointer, it returns to the user
- * specified pc, psl.
+ * Send an interrupt (signal) to a process.
  */
 void
 sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code)
 {
 	struct proc *p;
 	struct thread *td;
-	struct trapframe *frame;
+	struct trapframe *tf;
 	struct sigacts *psp;
 	struct sigframe sf, *sfp;
-	u_int64_t sbs = 0;
-	int oonstack, rndfsize;
+	mcontext_t *mc;
+	u_int64_t sbs, sp;
+	int oonstack;
 
 	td = curthread;
 	p = td->td_proc;
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 	psp = p->p_sigacts;
 	mtx_assert(&psp->ps_mtx, MA_OWNED);
-	frame = td->td_frame;
-	oonstack = sigonstack(frame->tf_r[FRAME_SP]);
-	rndfsize = ((sizeof(sf) + 15) / 16) * 16;
-
-	/*
-	 * Make sure that we restore the entire trapframe after a
-	 * signal.
-	 */
-	frame->tf_flags &= ~FRAME_SYSCALL;
+	tf = td->td_frame;
+	sp = tf->tf_special.sp;
+	oonstack = sigonstack(sp);
+	sbs = 0;
 
 	/* save user context */
 	bzero(&sf, sizeof(struct sigframe));
@@ -773,33 +837,6 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code)
 	sf.sf_uc.uc_stack = p->p_sigstk;
 	sf.sf_uc.uc_stack.ss_flags = (p->p_flag & P_ALTSTACK)
 	    ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
-	sf.sf_uc.uc_mcontext.mc_flags = IA64_MC_FLAG_ONSTACK;
-	sf.sf_uc.uc_mcontext.mc_onstack = (oonstack) ? 1 : 0;
-
-	sf.sf_uc.uc_mcontext.mc_nat     = 0; /* XXX */
-	sf.sf_uc.uc_mcontext.mc_sp	= frame->tf_r[FRAME_SP];
-	sf.sf_uc.uc_mcontext.mc_ip	= (frame->tf_cr_iip
-					   | ((frame->tf_cr_ipsr >> 41) & 3));
-	sf.sf_uc.uc_mcontext.mc_cfm     = frame->tf_cr_ifs & ~(1<<31);
-	sf.sf_uc.uc_mcontext.mc_um      = frame->tf_cr_ipsr & 0x1fff;
-	sf.sf_uc.uc_mcontext.mc_ar_rsc  = frame->tf_ar_rsc;
-	sf.sf_uc.uc_mcontext.mc_ar_bsp  = frame->tf_ar_bspstore;
-	sf.sf_uc.uc_mcontext.mc_ar_rnat = frame->tf_ar_rnat;
-	sf.sf_uc.uc_mcontext.mc_ar_ccv  = frame->tf_ar_ccv;
-	sf.sf_uc.uc_mcontext.mc_ar_unat = frame->tf_ar_unat;
-	sf.sf_uc.uc_mcontext.mc_ar_fpsr = frame->tf_ar_fpsr;
-	sf.sf_uc.uc_mcontext.mc_ar_pfs  = frame->tf_ar_pfs;
-	sf.sf_uc.uc_mcontext.mc_pr      = frame->tf_pr;
-
-	bcopy(&frame->tf_b[0],
-	      &sf.sf_uc.uc_mcontext.mc_br[0],
-	      8 * sizeof(unsigned long));
-	sf.sf_uc.uc_mcontext.mc_gr[0] = 0;
-	bcopy(&frame->tf_r[0],
-	      &sf.sf_uc.uc_mcontext.mc_gr[1],
-	      31 * sizeof(unsigned long));
-
-	/* XXX mc_fr[] */
 
 	/*
 	 * Allocate and validate space for the signal handler
@@ -810,47 +847,46 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code)
 	 */
 	if ((p->p_flag & P_ALTSTACK) != 0 && !oonstack &&
 	    SIGISMEMBER(psp->ps_sigonstack, sig)) {
-		sbs = (u_int64_t) p->p_sigstk.ss_sp;
-		sfp = (struct sigframe *)((caddr_t)p->p_sigstk.ss_sp +
-		    p->p_sigstk.ss_size - rndfsize);
-		/*
-		 * Align sp and bsp.
-		 */
+		sbs = (u_int64_t)p->p_sigstk.ss_sp;
 		sbs = (sbs + 15) & ~15;
-		sfp = (struct sigframe *)((u_int64_t)sfp & ~15);
+		sfp = (struct sigframe *)(sbs + p->p_sigstk.ss_size);
 #if defined(COMPAT_43) || defined(COMPAT_SUNOS)
 		p->p_sigstk.ss_flags |= SS_ONSTACK;
 #endif
 	} else
-		sfp = (struct sigframe *)(frame->tf_r[FRAME_SP] - rndfsize);
+		sfp = (struct sigframe *)sp;
+	sfp = (struct sigframe *)((u_int64_t)(sfp - 1) & ~15);
+
+	/* Fill in the siginfo structure for POSIX handlers. */
+	if (SIGISMEMBER(psp->ps_siginfo, sig)) {
+		sf.sf_si.si_signo = sig;
+		sf.sf_si.si_code = code;
+		sf.sf_si.si_addr = (void*)tf->tf_special.ifa;
+		code = (u_int64_t)&sfp->sf_si;
+	}
+
 	mtx_unlock(&psp->ps_mtx);
 	PROC_UNLOCK(p);
 
-#ifdef DEBUG
-	if ((sigdebug & SDB_KSTACK) && p->p_pid == sigpid)
-		printf("sendsig(%d): sig %d ssp %p usp %p\n", p->p_pid,
-		       sig, &sf, sfp);
-#endif
-
-#if 0
-	/* save the floating-point state, if necessary, then copy it. */
-	ia64_fpstate_save(td, 1);
-	sf.sf_uc.uc_mcontext.mc_ownedfp = td->td_md.md_flags & MDP_FPUSED;
-	bcopy(&td->td_pcb->pcb_fp,
-	      (struct fpreg *)sf.sf_uc.uc_mcontext.mc_fpregs,
-	      sizeof(struct fpreg));
-	sf.sf_uc.uc_mcontext.mc_fp_control = td->td_pcb.pcb_fp_control;
-#endif
+	mc = &sf.sf_uc.uc_mcontext;
+	mc->mc_special = tf->tf_special;
+	mc->mc_scratch = tf->tf_scratch;
+	if ((tf->tf_flags & FRAME_SYSCALL) == 0) {
+		mc->mc_flags |= IA64_MC_FLAGS_SCRATCH_VALID;
+		mc->mc_scratch_fp = tf->tf_scratch_fp;
+		/*
+		 * XXX High FP. If the process has never used the high FP,
+		 * mark the high FP as valid (zero defaults). If the process
+		 * did use the high FP, then store them in the PCB if not
+		 * already there (ie get them from the CPU that has them)
+		 * and write them in the context.
+		 */
+	}
+	save_callee_saved(&mc->mc_preserved);
+	save_callee_saved_fp(&mc->mc_preserved_fp);
 
-	/*
-	 * copy the frame out to userland.
-	 */
-	if (copyout((caddr_t)&sf, (caddr_t)sfp, sizeof(sf)) != 0) {
-#ifdef DEBUG
-		if ((sigdebug & SDB_KSTACK) && p->p_pid == sigpid)
-			printf("sendsig(%d): copyout failed on sig %d\n",
-			       p->p_pid, sig);
-#endif
+	/* Copy the frame out to userland. */
+	if (copyout(&sf, sfp, sizeof(sf)) != 0) {
 		/*
 		 * Process has trashed its stack; give it an illegal
 		 * instruction to halt it in its tracks.
@@ -859,46 +895,32 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code)
 		sigexit(td, SIGILL);
 		return;
 	}
-#ifdef DEBUG
-	if (sigdebug & SDB_FOLLOW)
-		printf("sendsig(%d): sig %d sfp %p code %lx\n", p->p_pid, sig,
-		    sfp, code);
-#endif
+
+	if ((tf->tf_flags & FRAME_SYSCALL) == 0) {
+		tf->tf_special.psr &= ~IA64_PSR_RI;
+		tf->tf_special.iip = ia64_get_k5() +
+		    ((uint64_t)break_sigtramp - (uint64_t)ia64_gateway_page);
+	} else
+		tf->tf_special.rp = ia64_get_k5() +
+		    ((uint64_t)epc_sigtramp - (uint64_t)ia64_gateway_page);
 
 	/*
-	 * Set up the registers to return to sigcode.
+	 * Setup the trapframe to return to the signal trampoline. We pass
+	 * information to the trampoline in the following registers:
+	 *
+	 *	gp	new backing store or NULL
+	 *	r8	signal number
+	 *	r9	signal code or siginfo pointer
+	 *	r10	signal handler (function descriptor)
 	 */
-	frame->tf_cr_ipsr &= ~IA64_PSR_RI;
-	frame->tf_cr_iip = PS_STRINGS - (esigcode - sigcode);
-	frame->tf_r[FRAME_R1] = sig;
+	tf->tf_special.sp = (u_int64_t)sfp - 16;
+	tf->tf_special.gp = sbs;
+	tf->tf_scratch.gr8 = sig;
+	tf->tf_scratch.gr9 = code;
+	tf->tf_scratch.gr10 = (u_int64_t)catcher;
+
 	PROC_LOCK(p);
 	mtx_lock(&psp->ps_mtx);
-	if (SIGISMEMBER(psp->ps_siginfo, sig)) {
-		frame->tf_r[FRAME_R15] = (u_int64_t)&(sfp->sf_si);
-
-		/* Fill in POSIX parts */
-		sf.sf_si.si_signo = sig;
-		sf.sf_si.si_code = code;
-		sf.sf_si.si_addr = (void*)frame->tf_cr_ifa;
-	}
-	else
-		frame->tf_r[FRAME_R15] = code;
-
-	frame->tf_r[FRAME_SP] = (u_int64_t)sfp - 16;
-	frame->tf_r[FRAME_R14] = sig;
-	frame->tf_r[FRAME_R15] = (u_int64_t) &sfp->sf_si;
-	frame->tf_r[FRAME_R16] = (u_int64_t) &sfp->sf_uc;
-	frame->tf_r[FRAME_R17] = (u_int64_t)catcher;
-	frame->tf_r[FRAME_R18] = sbs;
-
-#ifdef DEBUG
-	if (sigdebug & SDB_FOLLOW)
-		printf("sendsig(%d): pc %lx, catcher %lx\n", p->p_pid,
-		    frame->tf_cr_iip, frame->tf_regs[FRAME_R4]);
-	if ((sigdebug & SDB_KSTACK) && p->p_pid == sigpid)
-		printf("sendsig(%d): sig %d returns\n",
-		    p->p_pid, sig);
-#endif
 }
 
 /*
@@ -919,92 +941,54 @@ sigreturn(struct thread *td,
 	} */ *uap)
 {
 	ucontext_t uc;
-	const ucontext_t *ucp;
-	struct pcb *pcb;
-	struct trapframe *frame = td->td_frame;
-	struct __mcontext *mcp;
+	struct trapframe *tf;
+	struct __mcontext *mc;
 	struct proc *p;
+	struct pcb *pcb;
 
-	ucp = uap->sigcntxp;
-	pcb = td->td_pcb;
+	tf = td->td_frame;
 	p = td->td_proc;
-
-#ifdef DEBUG
-	if (sigdebug & SDB_FOLLOW)
-	    printf("sigreturn: pid %d, scp %p\n", p->p_pid, ucp);
-#endif
+	pcb = td->td_pcb;
 
 	/*
 	 * Fetch the entire context structure at once for speed.
 	 * We don't use a normal argument to simplify RSE handling.
 	 */
-	if (copyin((caddr_t)frame->tf_r[FRAME_R4],
-		   (caddr_t)&uc, sizeof(ucontext_t)))
+	if (copyin(uap->sigcntxp, (caddr_t)&uc, sizeof(uc)))
 		return (EFAULT);
 
-	if (frame->tf_ndirty != 0) {
-	    printf("sigreturn: dirty user stacked registers\n");
-	}
+	/*
+	 * XXX make sure ndirty in the current trapframe is less than
+	 * 0x1f8 so that if we throw away the current register stack,
+	 * we have reached the bottom of the kernel register stack.
+	 * See also exec_setregs.
+	 */
 
 	/*
 	 * Restore the user-supplied information
 	 */
-	mcp = &uc.uc_mcontext;
-	bcopy(&mcp->mc_br[0], &frame->tf_b[0], 8*sizeof(u_int64_t));
-	bcopy(&mcp->mc_gr[1], &frame->tf_r[0], 31*sizeof(u_int64_t));
-	/* XXX mc_fr */
-
-	frame->tf_flags &= ~FRAME_SYSCALL;
-	frame->tf_cr_iip = mcp->mc_ip & ~15;
-	frame->tf_cr_ipsr &= ~IA64_PSR_RI;
-	switch (mcp->mc_ip & 15) {
-	case 1:
-		frame->tf_cr_ipsr |= IA64_PSR_RI_1;
-		break;
-	case 2:
-		frame->tf_cr_ipsr |= IA64_PSR_RI_2;
-		break;
+	mc = &uc.uc_mcontext;
+	tf->tf_special = mc->mc_special;
+	tf->tf_scratch = mc->mc_scratch;
+	if ((mc->mc_flags & IA64_MC_FLAGS_SCRATCH_VALID) != 0) {
+		tf->tf_scratch_fp = mc->mc_scratch_fp;
+		/* XXX high FP. */
 	}
-	frame->tf_cr_ipsr     = ((frame->tf_cr_ipsr & ~0x1fff)
-				 | (mcp->mc_um & 0x1fff));
-	frame->tf_pr          = mcp->mc_pr;
-	frame->tf_ar_rsc      = (mcp->mc_ar_rsc & 3) | 12; /* user, loadrs=0 */
-	frame->tf_ar_pfs      = mcp->mc_ar_pfs;
-	frame->tf_cr_ifs      = mcp->mc_cfm | (1UL<<63);
-	frame->tf_ar_bspstore = mcp->mc_ar_bsp;
-	frame->tf_ar_rnat     = mcp->mc_ar_rnat;
-	frame->tf_ndirty      = 0; /* assumes flushrs in sigcode */
-	frame->tf_ar_unat     = mcp->mc_ar_unat;
-	frame->tf_ar_ccv      = mcp->mc_ar_ccv;
-	frame->tf_ar_fpsr     = mcp->mc_ar_fpsr;
-
-	frame->tf_r[FRAME_SP] = mcp->mc_sp;
+	restore_callee_saved(&mc->mc_preserved);
+	restore_callee_saved_fp(&mc->mc_preserved_fp);
 
 	PROC_LOCK(p);
 #if defined(COMPAT_43) || defined(COMPAT_SUNOS)
-	if (uc.uc_mcontext.mc_onstack & 1)
+	if (sigonstack(tf->tf_special.sp))
 		p->p_sigstk.ss_flags |= SS_ONSTACK;
 	else
 		p->p_sigstk.ss_flags &= ~SS_ONSTACK;
 #endif
-
 	td->td_sigmask = uc.uc_sigmask;
 	SIG_CANTMASK(td->td_sigmask);
 	signotify(td);
 	PROC_UNLOCK(p);
 
-	/* XXX ksc.sc_ownedfp ? */
-	ia64_fpstate_drop(td);
-#if 0
-	bcopy((struct fpreg *)uc.uc_mcontext.mc_fpregs,
-	      &td->td_pcb->pcb_fp, sizeof(struct fpreg));
-	td->td_pcb->pcb_fp_control = uc.uc_mcontext.mc_fp_control;
-#endif
-
-#ifdef DEBUG
-	if (sigdebug & SDB_FOLLOW)
-		printf("sigreturn(%d): returns\n", p->p_pid);
-#endif
 	return (EJUSTRETURN);
 }
 
@@ -1052,56 +1036,84 @@ cpu_halt(void)
 }
 
 /*
- * Clear registers on exec
+ * Clear registers on exec.
  */
 void
 exec_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 {
-	struct trapframe *frame;
+	struct trapframe *tf;
+	uint64_t bspst, kstack, ndirty;
+	size_t rssz;
 
-	frame = td->td_frame;
+	tf = td->td_frame;
+	kstack = td->td_kstack;
 
 	/*
-	 * Make sure that we restore the entire trapframe after an
-	 * execve.
-	 */
-	frame->tf_flags &= ~FRAME_SYSCALL;
-
-	bzero(frame->tf_r, sizeof(frame->tf_r));
-	bzero(frame->tf_f, sizeof(frame->tf_f));
-	frame->tf_cr_iip = entry;
-	frame->tf_cr_ipsr = (IA64_PSR_IC
-			     | IA64_PSR_I
-			     | IA64_PSR_IT
-			     | IA64_PSR_DT
-			     | IA64_PSR_RT
-			     | IA64_PSR_DFH
-			     | IA64_PSR_BN
-			     | IA64_PSR_CPL_USER);
-	/*
-	 * Make sure that sp is aligned to a 16 byte boundary and
-	 * reserve 16 bytes of scratch space for _start.
+	 * RSE magic: We have ndirty registers of the process on the kernel
+	 * stack which don't belong to the new image. Discard them. Note
+	 * that for the "legacy" syscall support we need to keep 3 registers
+	 * worth of dirty bytes. These 3 registers are the initial arguments
+	 * to the newly executing program.
+	 * However, we cannot discard all the ndirty registers by simply
+	 * moving the kernel related registers to the bottom of the kernel
+	 * stack and lowering the current bspstore, because we get into
+	 * trouble with the NaT collections. We need to keep that in sync
+	 * with the registers. Hence, we can only copy a multiple of 512
+	 * bytes. Consequently, we may end up with some registers of the
+	 * previous image on the kernel stack. This we ignore by making
+	 * sure we mask-off the lower 9 bits of the bspstore value just
+	 * prior to saving it in ar.k6.
 	 */
-	frame->tf_r[FRAME_SP] = (stack & ~15) - 16;
+	ndirty = tf->tf_special.ndirty & ~0x1ff;
+	if (ndirty > 0) {
+		__asm __volatile("mov	ar.rsc=0;;");
+		__asm __volatile("mov	%0=ar.bspstore" : "=r"(bspst));
+		rssz = bspst - kstack - ndirty;
+		bcopy((void*)(kstack + ndirty), (void*)kstack, rssz);
+		bspst -= ndirty;
+		__asm __volatile("mov	ar.bspstore=%0;;" :: "r"(bspst));
+		__asm __volatile("mov	ar.rsc=3");
+		tf->tf_special.ndirty -= ndirty;
+	}
+	ndirty = tf->tf_special.ndirty;
 
-	/*
-	 * Write values for out0, out1 and out2 to the user's backing
-	 * store and arrange for them to be restored into the user's
-	 * initial register frame. Assumes that (bspstore & 0x1f8) <
-	 * 0x1e0.
-	 */
-	frame->tf_ar_bspstore = td->td_md.md_bspstore + 24;
-	suword((caddr_t) frame->tf_ar_bspstore - 24, stack);
-	suword((caddr_t) frame->tf_ar_bspstore - 16, ps_strings);
-	suword((caddr_t) frame->tf_ar_bspstore -  8, 0);
-	frame->tf_ndirty = 0;
-	frame->tf_cr_ifs = (1L<<63) | 3; /* sof=3, v=1 */
-
-	frame->tf_ar_rsc = 0xf;	/* user mode rsc */
-	frame->tf_ar_fpsr = IA64_FPSR_DEFAULT;
-
-	td->td_md.md_flags &= ~MDP_FPUSED;
-	ia64_fpstate_drop(td);
+	bzero(&tf->tf_special, sizeof(tf->tf_special));
+
+	if ((tf->tf_flags & FRAME_SYSCALL) == 0) {	/* break syscalls. */
+		bzero(&tf->tf_scratch, sizeof(tf->tf_scratch));
+		bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp));
+		tf->tf_special.iip = entry;
+		tf->tf_special.cfm = (1UL<<63) | (3UL<<7) | 3UL;
+		tf->tf_special.bspstore = td->td_md.md_bspstore;
+		tf->tf_special.ndirty = 24;
+		/*
+		 * Copy the arguments onto the kernel register stack so that
+		 * they get loaded by the loadrs instruction.
+		 */
+		*(uint64_t*)(kstack + ndirty - 24) = stack;
+		*(uint64_t*)(kstack + ndirty - 16) = ps_strings;
+		*(uint64_t*)(kstack + ndirty - 8) = 0;
+	} else {				/* epc syscalls (default). */
+		tf->tf_special.rp = entry;
+		tf->tf_special.pfs = (3UL<<62) | (3UL<<7) | 3UL;
+		tf->tf_special.bspstore = td->td_md.md_bspstore + 24;
+		/*
+		 * Write values for out0, out1 and out2 to the user's backing
+		 * store and arrange for them to be restored into the user's
+		 * initial register frame.
+		 * Assumes that (bspstore & 0x1f8) < 0x1e0.
+		 */
+		suword((caddr_t)tf->tf_special.bspstore - 24, stack);
+		suword((caddr_t)tf->tf_special.bspstore - 16, ps_strings);
+		suword((caddr_t)tf->tf_special.bspstore -  8, 0);
+	}
+
+	tf->tf_special.sp = (stack & ~15) - 16;
+	tf->tf_special.rsc = 0xf;
+	tf->tf_special.fpsr = IA64_FPSR_DEFAULT;
+	tf->tf_special.psr = IA64_PSR_IC | IA64_PSR_I | IA64_PSR_IT |
+	    IA64_PSR_DT | IA64_PSR_RT | IA64_PSR_DFH | IA64_PSR_BN |
+	    IA64_PSR_CPL_USER;
 }
 
 int
@@ -1124,16 +1136,17 @@ ptrace_set_pc(struct thread *td, unsigned long addr)
 		return (EINVAL);
 	}
 
-	td->td_frame->tf_cr_iip = addr & ~0x0FULL;
-	td->td_frame->tf_cr_ipsr = (td->td_frame->tf_cr_ipsr & ~IA64_PSR_RI) |
-	    slot;
+	td->td_frame->tf_special.iip = addr & ~0x0FULL;
+	td->td_frame->tf_special.psr =
+	    (td->td_frame->tf_special.psr & ~IA64_PSR_RI) | slot;
 	return (0);
 }
 
 int
 ptrace_single_step(struct thread *td)
 {
-	td->td_frame->tf_cr_ipsr |= IA64_PSR_SS;
+
+	td->td_frame->tf_special.psr |= IA64_PSR_SS;
 	return (0);
 }
 
@@ -1144,66 +1157,26 @@ ia64_pa_access(vm_offset_t pa)
 }
 
 int
-fill_regs(td, regs)
-	struct thread *td;
-	struct reg *regs;
+fill_regs(struct thread *td, struct reg *regs)
 {
-	bcopy(td->td_frame->tf_b, regs->r_br, sizeof(regs->r_br));
-	bcopy(td->td_frame->tf_r, regs->r_gr+1, sizeof(td->td_frame->tf_r));
-	/* TODO copy registers from the register stack. */
-
-	regs->r_cfm = td->td_frame->tf_cr_ifs;
-	regs->r_ip = td->td_frame->tf_cr_iip;
-	regs->r_ip |= (td->td_frame->tf_cr_ipsr & IA64_PSR_RI) >> 41;
-	regs->r_pr = td->td_frame->tf_pr;
-	regs->r_psr = td->td_frame->tf_cr_ipsr;
-	regs->r_ar_rsc = td->td_frame->tf_ar_rsc;
-	regs->r_ar_bsp = 0;		/* XXX */
-	regs->r_ar_bspstore = td->td_frame->tf_ar_bspstore;
-	regs->r_ar_rnat = td->td_frame->tf_ar_rnat;
-	regs->r_ar_ccv = td->td_frame->tf_ar_ccv;
-	regs->r_ar_unat = td->td_frame->tf_ar_unat;
-	regs->r_ar_fpsr = td->td_frame->tf_ar_fpsr;
-	regs->r_ar_pfs = td->td_frame->tf_ar_pfs;
-	regs->r_ar_lc = td->td_frame->tf_ar_lc;
-	regs->r_ar_ec = td->td_frame->tf_ar_ec;
+	struct trapframe *tf;
 
+	tf = td->td_frame;
+	regs->r_special = tf->tf_special;
+	regs->r_scratch = tf->tf_scratch;
+	/* XXX preserved */
 	return (0);
 }
 
 int
-set_regs(td, regs)
-	struct thread *td;
-	struct reg *regs;
+set_regs(struct thread *td, struct reg *regs)
 {
-	int error;
-
-	error = ptrace_set_pc(td, regs->r_ip);
-	if (error)
-		return (error);
-
-	td->td_frame->tf_cr_ipsr &= ~0x1FUL;	/* clear user mask */
-	td->td_frame->tf_cr_ipsr |= regs->r_psr & 0x1FUL;
-
-	td->td_frame->tf_pr = regs->r_pr;
-
-	/* XXX r_ar_bsp */
-
-	td->td_frame->tf_ar_rsc = regs->r_ar_rsc;
-	td->td_frame->tf_ar_pfs = regs->r_ar_pfs;
-	td->td_frame->tf_cr_ifs = regs->r_cfm;
-	td->td_frame->tf_ar_bspstore = regs->r_ar_bspstore;
-	td->td_frame->tf_ar_rnat = regs->r_ar_rnat;
-	td->td_frame->tf_ar_unat = regs->r_ar_unat;
-	td->td_frame->tf_ar_ccv = regs->r_ar_ccv;
-	td->td_frame->tf_ar_fpsr = regs->r_ar_fpsr;
-	td->td_frame->tf_ar_lc = regs->r_ar_lc;
-	td->td_frame->tf_ar_ec = regs->r_ar_ec;
-
-	bcopy(regs->r_br, td->td_frame->tf_b, sizeof(td->td_frame->tf_b));
-	bcopy(regs->r_gr+1, td->td_frame->tf_r, sizeof(td->td_frame->tf_r));
-	/* TODO copy registers to the register stack. */
+	struct trapframe *tf;
 
+	tf = td->td_frame;
+	tf->tf_special = regs->r_special;
+	tf->tf_scratch = regs->r_scratch;
+	/* XXX preserved */
 	return (0);
 }
 
@@ -1222,67 +1195,105 @@ set_dbregs(struct thread *td, struct dbreg *dbregs)
 }
 
 int
-fill_fpregs(td, fpregs)
-	struct thread *td;
-	struct fpreg *fpregs;
+fill_fpregs(struct thread *td, struct fpreg *fpregs)
 {
+	struct trapframe *frame = td->td_frame;
 	struct pcb *pcb = td->td_pcb;
 
-	/*
-	 * XXX - The PCB pointer should not point to the actual PCB,
-	 * because it will not contain the preserved registers of
-	 * the program being debugged. Instead, it should point to
-	 * a PCB constructed by unwinding all the way up to the
-	 * IVT handler.
-	 */
-	bcopy(pcb->pcb_f + PCB_F2, fpregs->fpr_regs + 2,
-	    sizeof(pcb->pcb_f[0]) * 4);
+	/* Save the high FP registers. */
+	ia64_highfp_save(td);
 
-	bcopy(td->td_frame->tf_f, fpregs->fpr_regs + 6,
-	    sizeof(td->td_frame->tf_f));
+	fpregs->fpr_scratch = frame->tf_scratch_fp;
+	/* XXX preserved_fp */
+	fpregs->fpr_high = pcb->pcb_high_fp;
+	return (0);
+}
 
-	bcopy(pcb->pcb_f + PCB_F16, fpregs->fpr_regs + 16,
-	    sizeof(pcb->pcb_f[0]) * 16);
+int
+set_fpregs(struct thread *td, struct fpreg *fpregs)
+{
+	struct trapframe *frame = td->td_frame;
+	struct pcb *pcb = td->td_pcb;
 
-	ia64_fpstate_save(td, 0);
-	bcopy(pcb->pcb_highfp, fpregs->fpr_regs + 32,
-	    sizeof(td->td_pcb->pcb_highfp));
+	/* Throw away the high FP registers (should be redundant). */
+	ia64_highfp_drop(td);
 
+	frame->tf_scratch_fp = fpregs->fpr_scratch;
+	/* XXX preserved_fp */
+	pcb->pcb_high_fp = fpregs->fpr_high;
 	return (0);
 }
 
+/*
+ * High FP register functions.
+ * XXX no synchronization yet.
+ */
+
 int
-set_fpregs(td, fpregs)
-	struct thread *td;
-	struct fpreg *fpregs;
+ia64_highfp_drop(struct thread *td)
 {
-	struct pcb *pcb = td->td_pcb;
+	struct pcb *pcb;
+	struct pcpu *cpu;
+	struct thread *thr;
 
-	/*
-	 * XXX - The PCB pointer should not point to the actual PCB,
-	 * because it will not contain the preserved registers of
-	 * the program being debugged. Instead, it should point to
-	 * a PCB constructed by unwinding all the way up to the
-	 * IVT handler.
-	 * XXX - An additional complication here is that we need to
-	 * have the actual location of where the values should be
-	 * stored as well. Some values may still reside in registers,
-	 * while other may have been saved somewhere.
-	 */
-	bcopy(fpregs->fpr_regs + 2, pcb->pcb_f + PCB_F2,
-	    sizeof(pcb->pcb_f[0]) * 4);
+	pcb = td->td_pcb;
+	cpu = pcb->pcb_fpcpu;
+	if (cpu == NULL)
+		return (0);
+	pcb->pcb_fpcpu = NULL;
+	thr = cpu->pc_fpcurthread;
+	cpu->pc_fpcurthread = NULL;
+
+	/* Post-mortem sanity checking. */
+	KASSERT(thr == td, ("Inconsistent high FP state"));
+	return (1);
+}
 
-	bcopy(fpregs->fpr_regs + 6, td->td_frame->tf_f,
-	    sizeof(td->td_frame->tf_f));
+int
+ia64_highfp_load(struct thread *td)
+{
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+	KASSERT(pcb->pcb_fpcpu == NULL, ("FP race on thread"));
+	KASSERT(PCPU_GET(fpcurthread) == NULL, ("FP race on pcpu"));
+	restore_high_fp(&pcb->pcb_high_fp);
+	PCPU_SET(fpcurthread, td);
+	pcb->pcb_fpcpu = pcpup;
+	return (1);
+}
 
-	bcopy(fpregs->fpr_regs + 16, pcb->pcb_f + PCB_F16,
-	    sizeof(pcb->pcb_f[0]) * 16);
+int
+ia64_highfp_save(struct thread *td)
+{
+	struct pcb *pcb;
+	struct pcpu *cpu;
+	struct thread *thr;
 
-	ia64_fpstate_drop(td);
-	bcopy(fpregs->fpr_regs + 32, pcb->pcb_highfp,
-	    sizeof(td->td_pcb->pcb_highfp));
+	/* Don't save if the high FP registers weren't modified. */
+	if ((td->td_frame->tf_special.psr & IA64_PSR_MFH) == 0)
+		return (ia64_highfp_drop(td));
 
-	return (0);
+	pcb = td->td_pcb;
+	cpu = pcb->pcb_fpcpu;
+	if (cpu == NULL)
+		return (0);
+#ifdef SMP
+	if (cpu != pcpup) {
+		ipi_send(cpu->pc_lid, IPI_HIGH_FP);
+		while (pcb->pcb_fpcpu != cpu)
+			DELAY(100);
+		return (1);
+	}
+#endif
+	save_high_fp(&pcb->pcb_high_fp);
+	pcb->pcb_fpcpu = NULL;
+	thr = cpu->pc_fpcurthread;
+	cpu->pc_fpcurthread = NULL;
+
+	/* Post-mortem sanity cxhecking. */
+	KASSERT(thr == td, ("Inconsistent high FP state"));
+	return (1);
 }
 
 #ifndef DDB
@@ -1313,76 +1324,6 @@ SYSCTL_INT(_machdep, CPU_DISRTCSET, disable_rtc_set,
 SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock,
 	CTLFLAG_RW, &wall_cmos_clock, 0, "");
 
-void
-ia64_fpstate_check(struct thread *td)
-{
-	if ((td->td_frame->tf_cr_ipsr & IA64_PSR_DFH) == 0)
-		if (td != PCPU_GET(fpcurthread))
-			panic("ia64_fpstate_check: bogus");
-}
-
-/*
- * Save the high floating point state in the pcb. Use this to get
- * read-only access to the floating point state. If write is true, the
- * current fp process is cleared so that fp state can safely be
- * modified. The process will automatically reload the changed state
- * by generating a disabled fp trap.
- */
-void
-ia64_fpstate_save(struct thread *td, int write)
-{
-	if (td == PCPU_GET(fpcurthread)) {
-		/*
-		 * Save the state in the pcb.
-		 */
-		savehighfp(td->td_pcb->pcb_highfp);
-
-		if (write) {
-			td->td_frame->tf_cr_ipsr |= IA64_PSR_DFH;
-			PCPU_SET(fpcurthread, NULL);
-		}
-	}
-}
-
-/*
- * Relinquish ownership of the FP state. This is called instead of
- * ia64_save_fpstate() if the entire FP state is being changed
- * (e.g. on sigreturn).
- */
-void
-ia64_fpstate_drop(struct thread *td)
-{
-	if (td == PCPU_GET(fpcurthread)) {
-		td->td_frame->tf_cr_ipsr |= IA64_PSR_DFH;
-		PCPU_SET(fpcurthread, NULL);
-	}
-}
-
-/*
- * Switch the current owner of the fp state to p, reloading the state
- * from the pcb.
- */
-void
-ia64_fpstate_switch(struct thread *td)
-{
-	if (PCPU_GET(fpcurthread)) {
-		/*
-		 * Dump the old fp state if its valid.
-		 */
-		savehighfp(PCPU_GET(fpcurthread)->td_pcb->pcb_highfp);
-		PCPU_GET(fpcurthread)->td_frame->tf_cr_ipsr |= IA64_PSR_DFH;
-	}
-
-	/*
-	 * Remember the new FP owner and reload its state.
-	 */
-	PCPU_SET(fpcurthread, td);
-	restorehighfp(td->td_pcb->pcb_highfp);
-	td->td_frame->tf_cr_ipsr &= ~IA64_PSR_DFH;
-
-	td->td_md.md_flags |= MDP_FPUSED;
-}
-
 /*
  * Utility functions for manipulating instruction bundles.
  */
@@ -1407,47 +1348,8 @@ ia64_pack_bundle(u_int64_t *lowp, u_int64_t *highp,
 	*highp = high;
 }
 
-static int
-rse_slot(u_int64_t *bsp)
-{
-	return ((u_int64_t) bsp >> 3) & 0x3f;
-}
-
-/*
- * Return the address of register regno (regno >= 32) given that bsp
- * points at the base of the register stack frame.
- */
-u_int64_t *
-ia64_rse_register_address(u_int64_t *bsp, int regno)
-{
-	int off = regno - 32;
-	u_int64_t rnats = (rse_slot(bsp) + off) / 63;
-	return bsp + off + rnats;
-}
-
-/*
- * Calculate the base address of the previous frame given that the
- * current frame's locals area is 'size'.
- */
-u_int64_t *
-ia64_rse_previous_frame(u_int64_t *bsp, int size)
-{
-	int slot = rse_slot(bsp);
-	int rnats = 0;
-	int count = size;
-
-	while (count > slot) {
-		count -= 63;
-		rnats++;
-		slot = 63;
-	}
-	return bsp - size - rnats;
-}
-
-
 intptr_t
 casuptr(intptr_t *p, intptr_t old, intptr_t new)
 {
 	return (-1);
 }
-
diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c
index 43c06a7..f06e0cc 100644
--- a/sys/ia64/ia64/mp_machdep.c
+++ b/sys/ia64/ia64/mp_machdep.c
@@ -56,6 +56,8 @@
 #include <machine/fpu.h>
 #include <i386/include/specialreg.h>
 
+MALLOC_DECLARE(M_PMAP);
+
 void ia64_ap_startup(void);
 
 extern vm_offset_t vhpt_base, vhpt_size;
@@ -69,36 +71,37 @@ extern u_int64_t ia64_lapic_address;
 int	mp_ipi_test = 0;
 
 /* Variables used by os_boot_rendez */
-volatile vm_offset_t ap_stack;
-volatile struct pcpu *ap_pcpu;
+void *ap_stack;
+struct pcpu *ap_pcpu;
 volatile int ap_delay;
 volatile int ap_awake;
 volatile int ap_spin;
 
-static void ipi_send(u_int64_t, int);
 static void cpu_mp_unleash(void *);
 
 void
 ia64_ap_startup(void)
 {
+	ap_awake = 1;
+	ap_delay = 0;
+
 	__asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
 	    "r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1));
 
-	ia64_set_fpsr(IA64_FPSR_DEFAULT);
+	pcpup = ap_pcpu;
+	ia64_set_k4((intptr_t)pcpup);
 
-	/*
-	 * Set ia32 control registers.
-	 */
-	ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM|CR4_FXSR) << 32));
+	map_pal_code();
+	map_port_space();
+	map_gateway_page();
 
-	ap_awake = 1;
-	ap_delay = 0;
+	ia64_set_fpsr(IA64_FPSR_DEFAULT);
 
 	/* Wait until it's time for us to be unleashed */
 	while (ap_spin)
 		/* spin */;
 
-	__asm __volatile("ssm psr.ic|psr.i;; srlz.i;;");
+	__asm __volatile("ssm psr.i;; srlz.d;;");
 
 	/*
 	 * Get and save the CPU specific MCA records. Should we get the
@@ -122,8 +125,8 @@ ia64_ap_startup(void)
 	ia64_set_itm(ia64_get_itc() + itm_reload);
 	ia64_set_itv(CLOCK_VECTOR);
 	ia64_set_tpr(0);
-	cpu_throw();
-	panic("ia64_ap_startup: cpu_throw() returned");
+	cpu_throw(NULL, choosethread());
+	/* NOTREACHED */
 }
 
 int
@@ -213,22 +216,8 @@ cpu_mp_start()
 		pc->pc_current_pmap = kernel_pmap;
 		pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask;
 		if (pc->pc_cpuid > 0) {
-			void *ks;
-
-			/*
-			 * Use contigmalloc for stack so that we can
-			 * use a region 7 address for it which makes
-			 * it impossible to accidentally lose when
-			 * recording a trapframe.
-			 */
-			ks = contigmalloc(KSTACK_PAGES * PAGE_SIZE, M_TEMP,
-					  M_WAITOK,
-					  0ul,
-					  256*1024*1024 - 1,
-					  PAGE_SIZE,
-					  256*1024*1024);
-
-			ap_stack = IA64_PHYS_TO_RR7(ia64_tpa((u_int64_t)ks));
+			ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP,
+			    M_WAITOK);
 			ap_pcpu = pc;
 			ap_delay = 2000;
 			ap_awake = 0;
@@ -344,7 +333,7 @@ ipi_self(int ipi)
  * cr.lid (CR64) contents of the target processor. Only the id and eid
  * fields are used here.
  */
-static void
+void
 ipi_send(u_int64_t lid, int ipi)
 {
 	volatile u_int64_t *pipi;
diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c
index dcb49b9..998d49f 100644
--- a/sys/ia64/ia64/pmap.c
+++ b/sys/ia64/ia64/pmap.c
@@ -121,9 +121,13 @@
 
 #include <sys/user.h>
 
+#include <machine/cpu.h>
 #include <machine/pal.h>
 #include <machine/md_var.h>
 
+/* XXX move to a header. */
+extern u_int64_t ia64_gateway_page[];
+
 MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures");
 
 #ifndef KSTACK_MAX_PAGES
@@ -201,7 +205,7 @@ vm_offset_t vhpt_base, vhpt_size;
  * ia64_lptes. This gives us up to 2Gb of kernel virtual space.
  */
 static int nkpt;
-static struct ia64_lpte **kptdir;
+struct ia64_lpte **ia64_kptdir;
 #define KPTE_DIR_INDEX(va) \
 	((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1))
 #define KPTE_PTE_INDEX(va) \
@@ -369,12 +373,13 @@ pmap_bootstrap()
 	/*
 	 * Allocate some memory for initial kernel 'page tables'.
 	 */
-	kptdir = (struct ia64_lpte **) pmap_steal_memory(PAGE_SIZE);
+	ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE);
 	for (i = 0; i < NKPT; i++) {
-		kptdir[i] = (struct ia64_lpte *) pmap_steal_memory(PAGE_SIZE);
+		ia64_kptdir[i] = (void*)pmap_steal_memory(PAGE_SIZE);
 	}
 	nkpt = NKPT;
-	kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS;
+	kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS -
+	    VM_GATEWAY_SIZE;
 
 	avail_start = phys_avail[0];
 	for (i = 0; phys_avail[i+2]; i+= 2) ;
@@ -497,6 +502,8 @@ pmap_bootstrap()
 	 * Clear out any random TLB entries left over from booting.
 	 */
 	pmap_invalidate_all(kernel_pmap);
+
+	map_gateway_page();
 }
 
 void *
@@ -754,27 +761,14 @@ pmap_track_modified(vm_offset_t va)
 void
 pmap_new_thread(struct thread *td, int pages)
 {
-	vm_offset_t *ks;
 
 	/* Bounds check */
 	if (pages <= 1)
 		pages = KSTACK_PAGES;
 	else if (pages > KSTACK_MAX_PAGES)
 		pages = KSTACK_MAX_PAGES;
-
-	/*
-	 * Use contigmalloc for user area so that we can use a region
-	 * 7 address for it which makes it impossible to accidentally
-	 * lose when recording a trapframe.
-	 */
-	ks = contigmalloc(pages * PAGE_SIZE, M_PMAP, M_WAITOK, 0ul,
-	    256*1024*1024 - 1, PAGE_SIZE, 256*1024*1024);
-	if (ks == NULL)
-		panic("pmap_new_thread: could not contigmalloc %d pages\n",
-		    pages);
-
-	td->td_md.md_kstackvirt = ks;
-	td->td_kstack = IA64_PHYS_TO_RR7(ia64_tpa((u_int64_t)ks));
+	td->td_kstack = (vm_offset_t)malloc(pages * PAGE_SIZE, M_PMAP,
+	    M_WAITOK);
 	td->td_kstack_pages = pages;
 }
 
@@ -785,12 +779,10 @@ pmap_new_thread(struct thread *td, int pages)
 void
 pmap_dispose_thread(struct thread *td)
 {
-	int pages;
 
-	pages = td->td_kstack_pages;
-	contigfree(td->td_md.md_kstackvirt, pages * PAGE_SIZE, M_PMAP);
-	td->td_md.md_kstackvirt = NULL;
+	free((void*)td->td_kstack, M_PMAP);
 	td->td_kstack = 0;
+	td->td_kstack_pages = 0;
 }
 
 /*
@@ -800,16 +792,9 @@ void
 pmap_new_altkstack(struct thread *td, int pages)
 {
 
-	/*
-	 * Shuffle the original stack. Save the virtual kstack address
-	 * instead of the physical address because 1) we can derive the
-	 * physical address from the virtual address and 2) we need the
-	 * virtual address in pmap_dispose_thread.
-	 */
+	td->td_altkstack = td->td_kstack;
 	td->td_altkstack_obj = td->td_kstack_obj;
-	td->td_altkstack = (vm_offset_t)td->td_md.md_kstackvirt;
 	td->td_altkstack_pages = td->td_kstack_pages;
-
 	pmap_new_thread(td, pages);
 }
 
@@ -818,13 +803,7 @@ pmap_dispose_altkstack(struct thread *td)
 {
 
 	pmap_dispose_thread(td);
-
-	/*
-	 * Restore the original kstack. Note that td_altkstack holds the
-	 * virtual kstack address of the previous kstack.
-	 */
-	td->td_md.md_kstackvirt = (void*)td->td_altkstack;
-	td->td_kstack = IA64_PHYS_TO_RR7(ia64_tpa(td->td_altkstack));
+	td->td_kstack = td->td_altkstack;
 	td->td_kstack_obj = td->td_altkstack_obj;
 	td->td_kstack_pages = td->td_altkstack_pages;
 	td->td_altkstack = 0;
@@ -938,7 +917,7 @@ pmap_growkernel(vm_offset_t addr)
 		ptepage = (struct ia64_lpte *)
 		    IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg));
 		bzero(ptepage, PAGE_SIZE);
-		kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage;
+		ia64_kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage;
 
 		nkpt++;
 		kernel_vm_end += PAGE_SIZE * NKPTEPG;
@@ -1210,7 +1189,7 @@ pmap_find_kpte(vm_offset_t va)
 		("kernel mapping 0x%lx not in region 5", va));
 	KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG),
 		("kernel mapping 0x%lx out of range", va));
-	return &kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)];
+	return (&ia64_kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]);
 }
 
 /*
@@ -1355,6 +1334,7 @@ vm_paddr_t
 pmap_kextract(vm_offset_t va)
 {
 	struct ia64_lpte *pte;
+	vm_offset_t gwpage;
 
 	KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA"));
 
@@ -1362,6 +1342,11 @@ pmap_kextract(vm_offset_t va)
 	if (va >= IA64_RR_BASE(6))
 		return (IA64_RR_MASK(va));
 
+	/* EPC gateway page? */
+	gwpage = (vm_offset_t)ia64_get_k5();
+	if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE)
+		return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page));
+
 	/* Bail out if the virtual address is beyond our limits. */
 	if (IA64_RR_MASK(va) >= nkpt * PAGE_SIZE * NKPTEPG)
 		return (0);
diff --git a/sys/ia64/ia64/support.S b/sys/ia64/ia64/support.S
index 18537f0..7403a01 100644
--- a/sys/ia64/ia64/support.S
+++ b/sys/ia64/ia64/support.S
@@ -69,7 +69,6 @@
  *	ar.bsp	tranlated to new mode
  */
 ENTRY(ia64_change_mode, 0)
-
 	rsm	psr.i | psr.ic
 	mov	r19=ar.rsc		// save rsc while we change mode
 	tbit.nz	p6,p7=r14,17		// physical or virtual ?
@@ -106,7 +105,6 @@ ENTRY(ia64_change_mode, 0)
 	
 2:	mov	ar.rsc=r19		// restore ar.rsc
 	br.ret.sptk.few rp		// now in new mode
-	
 END(ia64_change_mode)
 
 /*
@@ -121,7 +119,6 @@ END(ia64_change_mode)
  *	psr.i	cleared
  */
 ENTRY(ia64_physical_mode, 0)
-
 	mov	r14=psr
 	mov	ret0=psr
 	movl	r15=(IA64_PSR_I|IA64_PSR_IT|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFL|IA64_PSR_DFH)
@@ -133,7 +130,6 @@ ENTRY(ia64_physical_mode, 0)
 	or	ret0=ret0,r16		// make sure BN=1
 
 	br.cond.sptk.many ia64_change_mode
-
 END(ia64_physical_mode)
 
 /*
@@ -148,7 +144,6 @@ END(ia64_physical_mode)
  *
  */
 ENTRY(ia64_call_efi_physical, 6)
-
 	.prologue
 	.regstk	6,4,5,0
 	.save	ar.pfs,loc0
@@ -183,7 +178,6 @@ ENTRY(ia64_call_efi_physical, 6)
 	mov	ar.pfs=loc0
 	;;
 	br.ret.sptk.many rp
-
 END(ia64_call_efi_physical)
 	
 /**************************************************************************/
@@ -194,8 +188,7 @@ END(ia64_call_efi_physical)
 	
 ENTRY(suword64, 2)
 XENTRY(suword)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -218,12 +211,10 @@ XENTRY(suword)
 
 	mov	ret0=r0
 	br.ret.sptk.few rp
-
 END(suword64)
 	
 ENTRY(suword32, 2)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -246,12 +237,10 @@ ENTRY(suword32, 2)
 
 	mov	ret0=r0
 	br.ret.sptk.few rp
-
 END(suword32)
 	
 ENTRY(subyte, 2)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -274,13 +263,11 @@ ENTRY(subyte, 2)
 
 	mov	ret0=r0
 	br.ret.sptk.few rp
-
 END(subyte)
 
 ENTRY(fuword64, 1)
 XENTRY(fuword)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -302,12 +289,10 @@ XENTRY(fuword)
 	st8	[r15]=r0			// clean up
 
 	br.ret.sptk.few rp
-
 END(fuword64)
 
 ENTRY(fuword32, 1)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -329,12 +314,10 @@ ENTRY(fuword32, 1)
 	st8	[r15]=r0			// clean up
 
 	br.ret.sptk.few rp
-
 END(fuword32)
 
 ENTRY(fubyte, 1)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -356,7 +339,6 @@ ENTRY(fubyte, 1)
 	st8	[r15]=r0			// clean up
 
 	br.ret.sptk.few rp
-
 END(fubyte)
 	
 ENTRY(fusufault, 0)
@@ -410,7 +392,6 @@ ENTRY(copystr, 4)
 
 4:	mov	ret0=0			// return 0.
 	br.ret.sptk.few rp
-	
 END(copystr)
 
 ENTRY(copyinstr, 4)
@@ -422,7 +403,7 @@ ENTRY(copyinstr, 4)
 	mov	loc1=rp
 	.body
 
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that src addr
+	movl	loc2=VM_MAX_ADDRESS		// make sure that src addr
 	;; 
 	cmp.geu	p6,p0=in0,loc2			// is in user space.
 	;; 
@@ -450,54 +431,12 @@ ENTRY(copyinstr, 4)
 	mov	ar.pfs=loc0			// restore ar.pfs
 	mov	rp=loc1				// restore ra.
 	br.ret.sptk.few rp			// ret0 left over from copystr
-
 END(copyinstr)
 
-ENTRY(copyoutstr, 4)
-	.prologue
-	.regstk	4, 3, 4, 0
-	.save	ar.pfs,loc0
-	alloc	loc0=ar.pfs,4,3,4,0
-	.save	rp,loc1
-	mov	loc1=rp
-	.body
-
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that dest addr
-	;; 
-	cmp.geu	p6,p0=in1,loc2			// is in user space.
-	;; 
-(p6)	br.cond.spnt.few copyerr		// if it's not, error out.
-	movl	r14=copyerr			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	loc2=PCB_ONFAULT,r15
-	;;
-	st8	[loc2]=r14
-	;;
-	mov	out0=in0
-	mov	out1=in1
-	mov	out2=in2
-	mov	out3=in3
-	;;
-	br.call.sptk.few rp=copystr		// do the copy.
-	st8	[loc2]=r0			// kill the fault handler.
-	mov	ar.pfs=loc0			// restore ar.pfs
-	mov	rp=loc1				// restore ra.
-	br.ret.sptk.few rp			// ret0 left over from copystr
-
-END(copyoutstr)
-
 /*
  * Not the fastest bcopy in the world.
  */
 ENTRY(bcopy, 3)
-
 	mov	ret0=r0				// return zero for copy{in,out}
 	;; 
 	cmp.le	p6,p0=in2,r0			// bail if len <= 0
@@ -559,29 +498,25 @@ ENTRY(bcopy, 3)
 (p6)	br.cond.spnt.few 6b
 
 	br.ret.sptk.few rp
-
 END(bcopy)
 
 ENTRY(memcpy,3)
-	
 	mov	r14=in0 ;;
 	mov	in0=in1 ;;
 	mov	in1=r14
 	br.cond.sptk.few bcopy
-	
 END(memcpy)
 	
 ENTRY(copyin, 3)
-	
 	.prologue
-	.regstk	4, 3, 4, 0
+	.regstk	3, 3, 3, 0
 	.save	ar.pfs,loc0
-	alloc	loc0=ar.pfs,4,3,4,0
+	alloc	loc0=ar.pfs,3,3,3,0
 	.save	rp,loc1
 	mov	loc1=rp
 	.body
 
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that src addr
+	movl	loc2=VM_MAX_ADDRESS		// make sure that src addr
 	;; 
 	cmp.geu	p6,p0=in0,loc2			// is in user space.
 	;; 
@@ -608,20 +543,18 @@ ENTRY(copyin, 3)
 	mov	ar.pfs=loc0			// restore ar.pfs
 	mov	rp=loc1				// restore ra.
 	br.ret.sptk.few rp			// ret0 left over from bcopy
-	
 END(copyin)
 
 ENTRY(copyout, 3)
-	
 	.prologue
-	.regstk	4, 3, 4, 0
+	.regstk	3, 3, 3, 0
 	.save	ar.pfs,loc0
-	alloc	loc0=ar.pfs,4,3,4,0
+	alloc	loc0=ar.pfs,3,3,3,0
 	.save	rp,loc1
 	mov	loc1=rp
 	.body
 
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that dest addr
+	movl	loc2=VM_MAX_ADDRESS		// make sure that dest addr
 	;; 
 	cmp.geu	p6,p0=in1,loc2			// is in user space.
 	;; 
@@ -648,11 +581,9 @@ ENTRY(copyout, 3)
 	mov	ar.pfs=loc0			// restore ar.pfs
 	mov	rp=loc1				// restore ra.
 	br.ret.sptk.few rp			// ret0 left over from bcopy
-	
 END(copyout)
 
 ENTRY(copyerr, 0)
-
 	add	r14=PC_CURTHREAD,r13 ;;		// find curthread
 	ld8	r14=[r14] ;;
 	add	r14=TD_PCB,r14 ;;		// curthread->td_addr
@@ -662,5 +593,4 @@ ENTRY(copyerr, 0)
 	
 	mov	ret0=EFAULT			// return EFAULT
 	br.ret.sptk.few rp
-
 END(copyerr)
diff --git a/sys/ia64/ia64/support.s b/sys/ia64/ia64/support.s
index 18537f0..7403a01 100644
--- a/sys/ia64/ia64/support.s
+++ b/sys/ia64/ia64/support.s
@@ -69,7 +69,6 @@
  *	ar.bsp	tranlated to new mode
  */
 ENTRY(ia64_change_mode, 0)
-
 	rsm	psr.i | psr.ic
 	mov	r19=ar.rsc		// save rsc while we change mode
 	tbit.nz	p6,p7=r14,17		// physical or virtual ?
@@ -106,7 +105,6 @@ ENTRY(ia64_change_mode, 0)
 	
 2:	mov	ar.rsc=r19		// restore ar.rsc
 	br.ret.sptk.few rp		// now in new mode
-	
 END(ia64_change_mode)
 
 /*
@@ -121,7 +119,6 @@ END(ia64_change_mode)
  *	psr.i	cleared
  */
 ENTRY(ia64_physical_mode, 0)
-
 	mov	r14=psr
 	mov	ret0=psr
 	movl	r15=(IA64_PSR_I|IA64_PSR_IT|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFL|IA64_PSR_DFH)
@@ -133,7 +130,6 @@ ENTRY(ia64_physical_mode, 0)
 	or	ret0=ret0,r16		// make sure BN=1
 
 	br.cond.sptk.many ia64_change_mode
-
 END(ia64_physical_mode)
 
 /*
@@ -148,7 +144,6 @@ END(ia64_physical_mode)
  *
  */
 ENTRY(ia64_call_efi_physical, 6)
-
 	.prologue
 	.regstk	6,4,5,0
 	.save	ar.pfs,loc0
@@ -183,7 +178,6 @@ ENTRY(ia64_call_efi_physical, 6)
 	mov	ar.pfs=loc0
 	;;
 	br.ret.sptk.many rp
-
 END(ia64_call_efi_physical)
 	
 /**************************************************************************/
@@ -194,8 +188,7 @@ END(ia64_call_efi_physical)
 	
 ENTRY(suword64, 2)
 XENTRY(suword)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -218,12 +211,10 @@ XENTRY(suword)
 
 	mov	ret0=r0
 	br.ret.sptk.few rp
-
 END(suword64)
 	
 ENTRY(suword32, 2)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -246,12 +237,10 @@ ENTRY(suword32, 2)
 
 	mov	ret0=r0
 	br.ret.sptk.few rp
-
 END(suword32)
 	
 ENTRY(subyte, 2)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -274,13 +263,11 @@ ENTRY(subyte, 2)
 
 	mov	ret0=r0
 	br.ret.sptk.few rp
-
 END(subyte)
 
 ENTRY(fuword64, 1)
 XENTRY(fuword)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -302,12 +289,10 @@ XENTRY(fuword)
 	st8	[r15]=r0			// clean up
 
 	br.ret.sptk.few rp
-
 END(fuword64)
 
 ENTRY(fuword32, 1)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -329,12 +314,10 @@ ENTRY(fuword32, 1)
 	st8	[r15]=r0			// clean up
 
 	br.ret.sptk.few rp
-
 END(fuword32)
 
 ENTRY(fubyte, 1)
-
-	movl	r14=VM_MAXUSER_ADDRESS;;	// make sure address is ok
+	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
 	cmp.geu	p6,p0=in0,r14
 (p6)	br.dpnt.few fusufault
 
@@ -356,7 +339,6 @@ ENTRY(fubyte, 1)
 	st8	[r15]=r0			// clean up
 
 	br.ret.sptk.few rp
-
 END(fubyte)
 	
 ENTRY(fusufault, 0)
@@ -410,7 +392,6 @@ ENTRY(copystr, 4)
 
 4:	mov	ret0=0			// return 0.
 	br.ret.sptk.few rp
-	
 END(copystr)
 
 ENTRY(copyinstr, 4)
@@ -422,7 +403,7 @@ ENTRY(copyinstr, 4)
 	mov	loc1=rp
 	.body
 
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that src addr
+	movl	loc2=VM_MAX_ADDRESS		// make sure that src addr
 	;; 
 	cmp.geu	p6,p0=in0,loc2			// is in user space.
 	;; 
@@ -450,54 +431,12 @@ ENTRY(copyinstr, 4)
 	mov	ar.pfs=loc0			// restore ar.pfs
 	mov	rp=loc1				// restore ra.
 	br.ret.sptk.few rp			// ret0 left over from copystr
-
 END(copyinstr)
 
-ENTRY(copyoutstr, 4)
-	.prologue
-	.regstk	4, 3, 4, 0
-	.save	ar.pfs,loc0
-	alloc	loc0=ar.pfs,4,3,4,0
-	.save	rp,loc1
-	mov	loc1=rp
-	.body
-
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that dest addr
-	;; 
-	cmp.geu	p6,p0=in1,loc2			// is in user space.
-	;; 
-(p6)	br.cond.spnt.few copyerr		// if it's not, error out.
-	movl	r14=copyerr			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	loc2=PCB_ONFAULT,r15
-	;;
-	st8	[loc2]=r14
-	;;
-	mov	out0=in0
-	mov	out1=in1
-	mov	out2=in2
-	mov	out3=in3
-	;;
-	br.call.sptk.few rp=copystr		// do the copy.
-	st8	[loc2]=r0			// kill the fault handler.
-	mov	ar.pfs=loc0			// restore ar.pfs
-	mov	rp=loc1				// restore ra.
-	br.ret.sptk.few rp			// ret0 left over from copystr
-
-END(copyoutstr)
-
 /*
  * Not the fastest bcopy in the world.
  */
 ENTRY(bcopy, 3)
-
 	mov	ret0=r0				// return zero for copy{in,out}
 	;; 
 	cmp.le	p6,p0=in2,r0			// bail if len <= 0
@@ -559,29 +498,25 @@ ENTRY(bcopy, 3)
 (p6)	br.cond.spnt.few 6b
 
 	br.ret.sptk.few rp
-
 END(bcopy)
 
 ENTRY(memcpy,3)
-	
 	mov	r14=in0 ;;
 	mov	in0=in1 ;;
 	mov	in1=r14
 	br.cond.sptk.few bcopy
-	
 END(memcpy)
 	
 ENTRY(copyin, 3)
-	
 	.prologue
-	.regstk	4, 3, 4, 0
+	.regstk	3, 3, 3, 0
 	.save	ar.pfs,loc0
-	alloc	loc0=ar.pfs,4,3,4,0
+	alloc	loc0=ar.pfs,3,3,3,0
 	.save	rp,loc1
 	mov	loc1=rp
 	.body
 
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that src addr
+	movl	loc2=VM_MAX_ADDRESS		// make sure that src addr
 	;; 
 	cmp.geu	p6,p0=in0,loc2			// is in user space.
 	;; 
@@ -608,20 +543,18 @@ ENTRY(copyin, 3)
 	mov	ar.pfs=loc0			// restore ar.pfs
 	mov	rp=loc1				// restore ra.
 	br.ret.sptk.few rp			// ret0 left over from bcopy
-	
 END(copyin)
 
 ENTRY(copyout, 3)
-	
 	.prologue
-	.regstk	4, 3, 4, 0
+	.regstk	3, 3, 3, 0
 	.save	ar.pfs,loc0
-	alloc	loc0=ar.pfs,4,3,4,0
+	alloc	loc0=ar.pfs,3,3,3,0
 	.save	rp,loc1
 	mov	loc1=rp
 	.body
 
-	movl	loc2=VM_MAXUSER_ADDRESS		// make sure that dest addr
+	movl	loc2=VM_MAX_ADDRESS		// make sure that dest addr
 	;; 
 	cmp.geu	p6,p0=in1,loc2			// is in user space.
 	;; 
@@ -648,11 +581,9 @@ ENTRY(copyout, 3)
 	mov	ar.pfs=loc0			// restore ar.pfs
 	mov	rp=loc1				// restore ra.
 	br.ret.sptk.few rp			// ret0 left over from bcopy
-	
 END(copyout)
 
 ENTRY(copyerr, 0)
-
 	add	r14=PC_CURTHREAD,r13 ;;		// find curthread
 	ld8	r14=[r14] ;;
 	add	r14=TD_PCB,r14 ;;		// curthread->td_addr
@@ -662,5 +593,4 @@ ENTRY(copyerr, 0)
 	
 	mov	ret0=EFAULT			// return EFAULT
 	br.ret.sptk.few rp
-
 END(copyerr)
diff --git a/sys/ia64/ia64/trap.c b/sys/ia64/ia64/trap.c
index bccba0e..cad006b 100644
--- a/sys/ia64/ia64/trap.c
+++ b/sys/ia64/ia64/trap.c
@@ -46,6 +46,7 @@
 #include <sys/sysent.h>
 #include <sys/syscall.h>
 #include <sys/pioctl.h>
+#include <sys/sysctl.h>
 #include <vm/vm.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_page.h>
@@ -61,6 +62,9 @@
 #include <machine/pal.h>
 #include <machine/fpu.h>
 #include <machine/efi.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 
 #ifdef KTRACE
 #include <sys/uio.h>
@@ -71,7 +75,13 @@
 #include <ddb/ddb.h>
 #endif
 
+static int print_usertrap = 0;
+SYSCTL_INT(_machdep, CPU_UNALIGNED_PRINT, print_usertrap,
+        CTLFLAG_RW, &print_usertrap, 0, "");
+
 extern int unaligned_fixup(struct trapframe *framep, struct thread *td);
+
+static void break_syscall(struct trapframe *tf);
 static void ia32_syscall(struct trapframe *framep);
 
 /*
@@ -93,12 +103,12 @@ typedef struct {
  * these there.  The rest of the registers are "live"
  */
 typedef struct {
-	u_int64_t	bitmask_low64;		/* f63 - f2 */
-	u_int64_t	bitmask_high64;		/* f127 - f64 */
-	struct ia64_fpreg *fp_low_preserved;	/* f2 - f5 */
-	struct ia64_fpreg *fp_low_volatile;	/* f6 - f15 */
-	struct ia64_fpreg *fp_high_preserved;	/* f16 - f31 */
-	struct ia64_fpreg *fp_high_volatile;	/* f32 - f127 */
+	u_int64_t	bitmask_low64;			/* f63 - f2 */
+	u_int64_t	bitmask_high64;			/* f127 - f64 */
+	struct _ia64_fpreg *fp_low_preserved;		/* f2 - f5 */
+	struct _ia64_fpreg *fp_low_volatile;		/* f6 - f15 */
+	struct _ia64_fpreg *fp_high_preserved;		/* f16 - f31 */
+	struct _ia64_fpreg *fp_high_volatile;		/* f32 - f127 */
 } FP_STATE;
 
 #ifdef WITNESS
@@ -271,7 +281,7 @@ static void printisr(u_int64_t isr)
 }
 
 static void
-printtrap(int vector, int imm, struct trapframe *framep, int isfatal, int user)
+printtrap(int vector, struct trapframe *framep, int isfatal, int user)
 {
 	printf("\n");
 	printf("%s %s trap (cpu %d):\n", isfatal? "fatal" : "handled",
@@ -279,16 +289,15 @@ printtrap(int vector, int imm, struct trapframe *framep, int isfatal, int user)
 	printf("\n");
 	printf("    trap vector = 0x%x (%s)\n",
 	       vector, ia64_vector_names[vector]);
-	printf("    cr.iip      = 0x%lx\n", framep->tf_cr_iip);
-	printf("    cr.ipsr     = 0x%lx (", framep->tf_cr_ipsr);
-	printpsr(framep->tf_cr_ipsr);
+	printf("    cr.iip      = 0x%lx\n", framep->tf_special.iip);
+	printf("    cr.ipsr     = 0x%lx (", framep->tf_special.psr);
+	printpsr(framep->tf_special.psr);
 	printf(")\n");
-	printf("    cr.isr      = 0x%lx (", framep->tf_cr_isr);
-	printisr(framep->tf_cr_isr);
+	printf("    cr.isr      = 0x%lx (", framep->tf_special.isr);
+	printisr(framep->tf_special.isr);
 	printf(")\n");
-	printf("    cr.ifa      = 0x%lx\n", framep->tf_cr_ifa);
-	printf("    cr.iim      = 0x%x\n", imm);
-	if (framep->tf_cr_ipsr & IA64_PSR_IS) {
+	printf("    cr.ifa      = 0x%lx\n", framep->tf_special.ifa);
+	if (framep->tf_special.psr & IA64_PSR_IS) {
 		printf("    ar.cflg     = 0x%lx\n", ia64_get_cflg());
 		printf("    ar.csd      = 0x%lx\n", ia64_get_csd());
 		printf("    ar.ssd      = 0x%lx\n", ia64_get_ssd());
@@ -301,34 +310,57 @@ printtrap(int vector, int imm, struct trapframe *framep, int isfatal, int user)
 }
 
 /*
+ *
+ */
+int
+do_ast(struct trapframe *tf)
+{
+
+	disable_intr();
+	while (curthread->td_flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)) {
+		enable_intr();
+		ast(tf);
+		disable_intr();
+	}
+	/*
+	 * Keep interrupts disabled. We return r10 as a favor to the EPC
+	 * syscall code so that it can quicky determine if the syscall
+	 * needs to be restarted or not.
+	 */
+	return (tf->tf_scratch.gr10);
+}
+
+/*
  * Trap is called from exception.s to handle most types of processor traps.
- * System calls are broken out for efficiency and ASTs are broken out
- * to make the code a bit cleaner and more representative of the
- * architecture.
  */
 /*ARGSUSED*/
 void
-trap(int vector, int imm, struct trapframe *framep)
+trap(int vector, struct trapframe *framep)
 {
-	struct thread *td;
 	struct proc *p;
-	int i;
+	struct thread *td;
 	u_int64_t ucode;
+	int i, user;
 	u_int sticks;
-	int user;
 
-	cnt.v_trap++;
+	user = ((framep->tf_special.psr & IA64_PSR_CPL) == IA64_PSR_CPL_USER);
+
+	/* Short-circuit break instruction based system calls. */
+	if (vector == IA64_VEC_BREAK && user &&
+	    framep->tf_special.ifa == 0x100000) {
+		break_syscall(framep);
+		return;
+	}
+
+	/* Sanitize the FP state in case the user has trashed it. */
+	ia64_set_fpsr(IA64_FPSR_DEFAULT);
+
+	atomic_add_int(&cnt.v_trap, 1);
+
 	td = curthread;
 	p = td->td_proc;
 	ucode = 0;
 
-	/*
-	 * Make sure we have a sane floating-point state in case the
-	 * user has trashed it.
-	 */
-	ia64_set_fpsr(IA64_FPSR_DEFAULT);
-
-	user = ((framep->tf_cr_ipsr & IA64_PSR_CPL) == IA64_PSR_CPL_USER);
 	if (user) {
 		sticks = td->td_sticks;
 		td->td_frame = framep;
@@ -341,19 +373,18 @@ trap(int vector, int imm, struct trapframe *framep)
 	}
 
 	switch (vector) {
-	case IA64_VEC_UNALIGNED_REFERENCE:
+
+	case IA64_VEC_UNALIGNED_REFERENCE: {
 		/*
 		 * If user-land, do whatever fixups, printing, and
 		 * signalling is appropriate (based on system-wide
 		 * and per-process unaligned-access-handling flags).
 		 */
 		if (user) {
-			mtx_lock(&Giant);
 			i = unaligned_fixup(framep, td);
-			mtx_unlock(&Giant);
 			if (i == 0)
 				goto out;
-			ucode = framep->tf_cr_ifa;	/* VA */
+			ucode = framep->tf_special.ifa;	/* VA */
 			break;
 		}
 
@@ -369,9 +400,9 @@ trap(int vector, int imm, struct trapframe *framep)
 		 * does cause an unaligned access it's a kernel bug.
 		 */
 		goto dopanic;
+	}
 
-	case IA64_VEC_FLOATING_POINT_FAULT:
-	{
+	case IA64_VEC_FLOATING_POINT_FAULT: {
 		FP_STATE fp_state;
 		FPSWA_RET fpswa_ret;
 		FPSWA_BUNDLE bundle;
@@ -385,7 +416,7 @@ trap(int vector, int imm, struct trapframe *framep)
 			break;
 		}
 		mtx_lock(&Giant);
-	        i = copyin((const void *)(framep->tf_cr_iip), &bundle, 16);
+		i = copyin((void *)(framep->tf_special.iip), &bundle, 16);
 		mtx_unlock(&Giant);
 		if (i) {
 			i = SIGBUS;		/* EFAULT, basically */
@@ -396,28 +427,28 @@ trap(int vector, int imm, struct trapframe *framep)
 		fp_state.bitmask_low64 = 0xffc0;	/* bits 6 - 15 */
 		fp_state.bitmask_high64 = 0x0;
 		fp_state.fp_low_preserved = NULL;
-		fp_state.fp_low_volatile = framep->tf_f;
+		fp_state.fp_low_volatile = &framep->tf_scratch_fp.fr6;
 		fp_state.fp_high_preserved = NULL;
 		fp_state.fp_high_volatile = NULL;
 		/* The docs are unclear.  Is Fpswa reentrant? */
 		fpswa_ret = fpswa_interface->Fpswa(1, &bundle,
-		    &framep->tf_cr_ipsr, &framep->tf_ar_fpsr,
-		    &framep->tf_cr_isr, &framep->tf_pr,
-		    &framep->tf_cr_ifs, &fp_state);
+		    &framep->tf_special.psr, &framep->tf_special.fpsr,
+		    &framep->tf_special.isr, &framep->tf_special.pr,
+		    &framep->tf_special.cfm, &fp_state);
 		if (fpswa_ret.status == 0) {
 			/* fixed.  update ipsr and iip to next insn */
 			int ei;
 
-			ei = (framep->tf_cr_isr >> 41) & 0x03;
+			ei = (framep->tf_special.isr >> 41) & 0x03;
 			if (ei == 0) {		/* no template for this case */
-				framep->tf_cr_ipsr &= ~IA64_ISR_EI;
-				framep->tf_cr_ipsr |= IA64_ISR_EI_1;
+				framep->tf_special.psr &= ~IA64_ISR_EI;
+				framep->tf_special.psr |= IA64_ISR_EI_1;
 			} else if (ei == 1) {	/* MFI or MFB */
-				framep->tf_cr_ipsr &= ~IA64_ISR_EI;
-				framep->tf_cr_ipsr |= IA64_ISR_EI_2;
+				framep->tf_special.psr &= ~IA64_ISR_EI;
+				framep->tf_special.psr |= IA64_ISR_EI_2;
 			} else if (ei == 2) {	/* MMF */
-				framep->tf_cr_ipsr &= ~IA64_ISR_EI;
-				framep->tf_cr_iip += 0x10;
+				framep->tf_special.psr &= ~IA64_ISR_EI;
+				framep->tf_special.iip += 0x10;
 			}
 			goto out;
 		} else if (fpswa_ret.status == -1) {
@@ -446,8 +477,7 @@ trap(int vector, int imm, struct trapframe *framep)
 		}
 	}
 
-	case IA64_VEC_FLOATING_POINT_TRAP:
-	{
+	case IA64_VEC_FLOATING_POINT_TRAP: {
 		FP_STATE fp_state;
 		FPSWA_RET fpswa_ret;
 		FPSWA_BUNDLE bundle;
@@ -461,7 +491,7 @@ trap(int vector, int imm, struct trapframe *framep)
 			break;
 		}
 		mtx_lock(&Giant);
-	        i = copyin((const void *)(framep->tf_cr_iip), &bundle, 16);
+		i = copyin((void *)(framep->tf_special.iip), &bundle, 16);
 		mtx_unlock(&Giant);
 		if (i) {
 			i = SIGBUS;			/* EFAULT, basically */
@@ -472,14 +502,14 @@ trap(int vector, int imm, struct trapframe *framep)
 		fp_state.bitmask_low64 = 0xffc0;	/* bits 6 - 15 */
 		fp_state.bitmask_high64 = 0x0;
 		fp_state.fp_low_preserved = NULL;
-		fp_state.fp_low_volatile = framep->tf_f;
+		fp_state.fp_low_volatile = &framep->tf_scratch_fp.fr6;
 		fp_state.fp_high_preserved = NULL;
 		fp_state.fp_high_volatile = NULL;
 		/* The docs are unclear.  Is Fpswa reentrant? */
 		fpswa_ret = fpswa_interface->Fpswa(0, &bundle,
-		    &framep->tf_cr_ipsr, &framep->tf_ar_fpsr,
-		    &framep->tf_cr_isr, &framep->tf_pr,
-		    &framep->tf_cr_ifs, &fp_state);
+		    &framep->tf_special.psr, &framep->tf_special.fpsr,
+		    &framep->tf_special.isr, &framep->tf_special.pr,
+		    &framep->tf_special.cfm, &fp_state);
 		if (fpswa_ret.status == 0) {
 			/* fixed */
 			/*
@@ -501,24 +531,97 @@ trap(int vector, int imm, struct trapframe *framep)
 		}
 	}
 
-	case IA64_VEC_DISABLED_FP:
+	case IA64_VEC_DISABLED_FP: {	/* High FP registers are disabled. */
+		struct pcpu *pcpu;
+		struct pcb *pcb;
+		struct thread *thr;
+
+		/* Always fatal in kernel. Should never happen. */
+		if (!user)
+			goto dopanic;
+
+		pcb = td->td_pcb;
+		pcpu = pcb->pcb_fpcpu;
+
+#if 0
+		printf("XXX: td %p: highfp on cpu %p\n", td, pcpu);
+#endif
+
 		/*
-		 * on exit from the kernel, if thread == fpcurthread,
-		 * FP is enabled.
+		 * The pcpu variable holds the address of the per-CPU
+		 * structure of the CPU currently holding this threads
+		 * high FP registers (or NULL if no CPU holds these
+		 * registers). We have to interrupt that CPU and wait
+		 * for it to have saved the registers.
 		 */
-		if (PCPU_GET(fpcurthread) == td) {
-			printf("trap: fp disabled for fpcurthread == %p", td);
-			goto dopanic;
+		if (pcpu != NULL) {
+			thr = pcpu->pc_fpcurthread;
+			KASSERT(thr == td, ("High FP state out of sync"));
+
+			if (pcpu == pcpup) {
+				/*
+				 * Short-circuit handling the trap when this
+				 * CPU already holds the high FP registers for
+				 * this thread. We really shouldn't get the
+				 * trap in the first place, but since it's
+				 * only a performance issue and not a
+				 * correctness issue, we emit a message for
+				 * now, enable the high FP registers and
+				 * return.
+				 */
+				printf("XXX: bogusly disabled high FP regs\n");
+				framep->tf_special.psr &= ~IA64_PSR_DFH;
+				goto out;
+			}
+#ifdef SMP
+			/*
+			 * Interrupt the other CPU so that it saves the high
+			 * FP registers of this thread. Note that this can
+			 * only happen for the SMP case.
+			 */
+			ipi_send(pcpu->pc_lid, IPI_HIGH_FP);
+#endif
+#ifdef DIAGNOSTICS
+		} else {
+			KASSERT(PCPU_GET(fpcurthread) != td,
+			    ("High FP state out of sync"));
+#endif
 		}
-	
-		ia64_fpstate_switch(td);
+
+		thr = PCPU_GET(fpcurthread);
+
+#if 0
+		printf("XXX: cpu %p: highfp belongs to td %p\n", pcpup, thr);
+#endif
+
+		/*
+		 * The thr variable holds the thread that owns the high FP
+		 * registers currently on this CPU. Free this CPU so that
+		 * we can load the current threads high FP registers.
+		 */
+		if (thr != NULL) {
+			KASSERT(thr != td, ("High FP state out of sync"));
+			pcb = thr->td_pcb;
+			KASSERT(pcb->pcb_fpcpu == pcpup,
+			    ("High FP state out of sync"));
+			ia64_highfp_save(thr);
+		}
+
+		/*
+		 * Wait for the other CPU to have saved out high FP
+		 * registers (if applicable).
+		 */
+		while (pcpu && pcpu->pc_fpcurthread == td);
+
+		ia64_highfp_load(td);
+		framep->tf_special.psr &= ~IA64_PSR_DFH;
 		goto out;
 		break;
+	}
 
 	case IA64_VEC_PAGE_NOT_PRESENT:
 	case IA64_VEC_INST_ACCESS_RIGHTS:
-	case IA64_VEC_DATA_ACCESS_RIGHTS:
-	{
+	case IA64_VEC_DATA_ACCESS_RIGHTS: {
 		vm_offset_t va;
 		struct vmspace *vm;
 		vm_map_t map;
@@ -526,7 +629,7 @@ trap(int vector, int imm, struct trapframe *framep)
 		int rv;
 
 		rv = 0; 
-		va = framep->tf_cr_ifa;
+		va = framep->tf_special.ifa;
 
 		/*
 		 * If it was caused by fuswintr or suswintr, just punt. Note
@@ -536,8 +639,8 @@ trap(int vector, int imm, struct trapframe *framep)
 		 */
 		if (!user && td != NULL && td->td_pcb->pcb_accessaddr == va &&
 		    td->td_pcb->pcb_onfault == (unsigned long)fswintrberr) {
-			framep->tf_cr_iip = td->td_pcb->pcb_onfault;
-			framep->tf_cr_ipsr &= ~IA64_PSR_RI;
+			framep->tf_special.iip = td->td_pcb->pcb_onfault;
+			framep->tf_special.psr &= ~IA64_PSR_RI;
 			td->td_pcb->pcb_onfault = 0;
 			goto out;
 		}
@@ -559,9 +662,9 @@ trap(int vector, int imm, struct trapframe *framep)
 			map = &vm->vm_map;
 		}
 
-		if (framep->tf_cr_isr & IA64_ISR_X)
+		if (framep->tf_special.isr & IA64_ISR_X)
 			ftype = VM_PROT_EXECUTE;
-		else if (framep->tf_cr_isr & IA64_ISR_W)
+		else if (framep->tf_special.isr & IA64_ISR_W)
 			ftype = VM_PROT_WRITE;
 		else
 			ftype = VM_PROT_READ;
@@ -597,8 +700,9 @@ trap(int vector, int imm, struct trapframe *framep)
 		if (!user) {
 			/* Check for copyin/copyout fault. */
 			if (td != NULL && td->td_pcb->pcb_onfault != 0) {
-				framep->tf_cr_iip = td->td_pcb->pcb_onfault;
-				framep->tf_cr_ipsr &= ~IA64_PSR_RI;
+				framep->tf_special.iip =
+				    td->td_pcb->pcb_onfault;
+				framep->tf_special.psr &= ~IA64_PSR_RI;
 				td->td_pcb->pcb_onfault = 0;
 				goto out;
 			}
@@ -609,10 +713,10 @@ trap(int vector, int imm, struct trapframe *framep)
 		break;
 	}
 
-	case IA64_VEC_SINGLE_STEP_TRAP:
-	case IA64_VEC_DEBUG:
-	case IA64_VEC_TAKEN_BRANCH_TRAP:
 	case IA64_VEC_BREAK:
+	case IA64_VEC_DEBUG:
+	case IA64_VEC_SINGLE_STEP_TRAP:
+	case IA64_VEC_TAKEN_BRANCH_TRAP: {
 		/*
 		 * These are always fatal in kernel, and should never happen.
 		 */
@@ -633,27 +737,29 @@ trap(int vector, int imm, struct trapframe *framep)
 		}
 		i = SIGTRAP;
 		break;
+	}
 
-	case IA64_VEC_GENERAL_EXCEPTION:
+	case IA64_VEC_GENERAL_EXCEPTION: {
 		if (user) {
 			ucode = vector;
 			i = SIGILL;
 			break;
 		}
 		goto dopanic;
+	}
 
 	case IA64_VEC_UNSUPP_DATA_REFERENCE:
-	case IA64_VEC_LOWER_PRIVILEGE_TRANSFER:
+	case IA64_VEC_LOWER_PRIVILEGE_TRANSFER: {
 		if (user) {
 			ucode = vector;
 			i = SIGBUS;
 			break;
 		}
 		goto dopanic;
+	}
 
-	case IA64_VEC_IA32_EXCEPTION:
-	{
-		u_int64_t isr = framep->tf_cr_isr;
+	case IA64_VEC_IA32_EXCEPTION: {
+		u_int64_t isr = framep->tf_special.isr;
 
 		switch ((isr >> 16) & 0xffff) {
 		case IA32_EXCEPTION_DIVIDE:
@@ -694,7 +800,7 @@ trap(int vector, int imm, struct trapframe *framep)
 			break;
 			
 		case IA32_EXCEPTION_ALIGNMENT_CHECK:
-			ucode = framep->tf_cr_ifa;	/* VA */
+			ucode = framep->tf_special.ifa;	/* VA */
 			i = SIGBUS;
 			break;
 			
@@ -709,33 +815,36 @@ trap(int vector, int imm, struct trapframe *framep)
 		break;
 	}
 
-	case IA64_VEC_IA32_INTERRUPT:
+	case IA64_VEC_IA32_INTERRUPT: {
 		/*
 		 * INT n instruction - probably a syscall.
 		 */
-		if (((framep->tf_cr_isr >> 16) & 0xffff) == 0x80) {
+		if (((framep->tf_special.isr >> 16) & 0xffff) == 0x80) {
 			ia32_syscall(framep);
 			goto out;
 		} else {
-			ucode = (framep->tf_cr_isr >> 16) & 0xffff;
+			ucode = (framep->tf_special.isr >> 16) & 0xffff;
 			i = SIGILL;
 			break;
 		}
+	}
 
-	case IA64_VEC_IA32_INTERCEPT:
+	case IA64_VEC_IA32_INTERCEPT: {
 		/*
 		 * Maybe need to emulate ia32 instruction.
 		 */
 		goto dopanic;
+	}
 
 	default:
 		goto dopanic;
 	}
 
-#ifdef DEBUG
-	printtrap(vector, imm, framep, 1, user);
-#endif
+	if (print_usertrap)
+		printtrap(vector, framep, 1, user);
+
 	trapsignal(td, i, ucode);
+
 out:
 	if (user) {
 		userret(td, framep, sticks);
@@ -743,73 +852,96 @@ out:
 #ifdef DIAGNOSTIC
 		cred_free_thread(td);
 #endif
+		do_ast(framep);
 	}
 	return;
 
 dopanic:
-	printtrap(vector, imm, framep, 1, user);
-
-	/* XXX dump registers */
-
+	printtrap(vector, framep, 1, user);
 #ifdef DDB
 	kdb_trap(vector, framep);
 #endif
-
 	panic("trap");
 }
 
+
+/*
+ * Handle break instruction based system calls.
+ */
+void
+break_syscall(struct trapframe *tf)
+{
+	uint64_t *bsp, *tfp;
+	uint64_t iip, psr;
+	int error, nargs;
+
+	/* Save address of break instruction. */
+	iip = tf->tf_special.iip;
+	psr = tf->tf_special.psr;
+
+	/* Advance to the next instruction. */
+	tf->tf_special.psr += IA64_PSR_RI_1;
+	if ((tf->tf_special.psr & IA64_PSR_RI) > IA64_PSR_RI_2) {
+		tf->tf_special.iip += 16;
+		tf->tf_special.psr &= ~IA64_PSR_RI;
+	}
+
+	/*
+	 * Copy the arguments on the register stack into the trapframe
+	 * to avoid having interleaved NaT collections.
+	 */
+	tfp = &tf->tf_scratch.gr16;
+	nargs = tf->tf_special.cfm & 0x7f;
+	bsp = (uint64_t*)(curthread->td_kstack + tf->tf_special.ndirty);
+	bsp -= (((uintptr_t)bsp & 0x1ff) < (nargs << 3)) ? (nargs + 1): nargs;
+	while (nargs--) {
+		*tfp++ = *bsp++;
+		if (((uintptr_t)bsp & 0x1ff) == 0x1f8)
+			bsp++;
+	}
+	error = syscall(tf);
+	if (error == ERESTART) {
+		tf->tf_special.iip = iip;
+		tf->tf_special.psr = psr;
+	}
+
+	do_ast(tf);
+}
+
 /*
  * Process a system call.
  *
- * System calls are strange beasts.  They are passed the syscall number
- * in r15, and the arguments in the registers (as normal).  They return
- * an error flag in r10 (if r10 != 0 on return, the syscall had an error),
- * and the return value (if any) in r8 and r9.
- *
- * The assembly stub takes care of moving the call number into a register
- * we can get to, and moves all of the argument registers into a stack 
- * buffer.  On return, it restores r8-r10 from the frame before
- * returning to the user process. 
+ * See syscall.s for details as to how we get here. In order to support
+ * the ERESTART case, we return the error to our caller. They deal with
+ * the hairy details.
  */
-void
-syscall(int code, u_int64_t *args, struct trapframe *framep)
+int
+syscall(struct trapframe *tf)
 {
 	struct sysent *callp;
-	struct thread *td;
 	struct proc *p;
-	int error = 0;
-	u_int64_t oldip, oldri;
+	struct thread *td;
+	u_int64_t *args;
+	int code, error;
 	u_int sticks;
 
-	cnt.v_syscall++;
+	code = tf->tf_scratch.gr15;
+	args = &tf->tf_scratch.gr16;
+
+	atomic_add_int(&cnt.v_syscall, 1);
+
 	td = curthread;
 	p = td->td_proc;
 
-	td->td_frame = framep;
+	td->td_frame = tf;
 	sticks = td->td_sticks;
 	if (td->td_ucred != p->p_ucred)
 		cred_update_thread(td);
-
-	/*
-	 * Skip past the break instruction. Remember old address in case
-	 * we have to restart.
-	 */
-	oldip = framep->tf_cr_iip;
-	oldri = framep->tf_cr_ipsr & IA64_PSR_RI;
-	framep->tf_cr_ipsr += IA64_PSR_RI_1;
-	if ((framep->tf_cr_ipsr & IA64_PSR_RI) > IA64_PSR_RI_2) {
-		framep->tf_cr_ipsr &= ~IA64_PSR_RI;
-		framep->tf_cr_iip += 16;
-	}
-			   
 	if (p->p_flag & P_THREADED)
 		thread_user_enter(p, td);
-#ifdef DIAGNOSTIC
-	ia64_fpstate_check(td);
-#endif
 
 	if (p->p_sysent->sv_prepsyscall) {
-		/* (*p->p_sysent->sv_prepsyscall)(framep, args, &code, &params); */
+		/* (*p->p_sysent->sv_prepsyscall)(tf, args, &code, &params); */
 		panic("prepsyscall");
 	} else {
 		/*
@@ -842,39 +974,33 @@ syscall(int code, u_int64_t *args, struct trapframe *framep)
 	if (KTRPOINT(td, KTR_SYSCALL))
 		ktrsyscall(code, (callp->sy_narg & SYF_ARGMASK), args);
 #endif
-	if (error == 0) {
-		td->td_retval[0] = 0;
-		td->td_retval[1] = 0;
 
-		STOPEVENT(p, S_SCE, (callp->sy_narg & SYF_ARGMASK));
+	td->td_retval[0] = 0;
+	td->td_retval[1] = 0;
+	tf->tf_scratch.gr10 = EJUSTRETURN;
 
-		error = (*callp->sy_call)(td, args);
-	}
+	STOPEVENT(p, S_SCE, (callp->sy_narg & SYF_ARGMASK));
 
+	error = (*callp->sy_call)(td, args);
 
-	switch (error) {
-	case 0:
-		framep->tf_r[FRAME_R8] = td->td_retval[0];
-		framep->tf_r[FRAME_R9] = td->td_retval[1];
-		framep->tf_r[FRAME_R10] = 0;
-		break;
-	case ERESTART:
-		framep->tf_cr_iip = oldip;
-		framep->tf_cr_ipsr =
-			(framep->tf_cr_ipsr & ~IA64_PSR_RI) | oldri;
-		break;
-	case EJUSTRETURN:
-		break;
-	default:
-		if (p->p_sysent->sv_errsize) {
-			if (error >= p->p_sysent->sv_errsize)
-				error = -1; /* XXX */
-			else
+	if (error != EJUSTRETURN) {
+		/*
+		 * Save the "raw" error code in r10. We use this to handle
+		 * syscall restarts (see do_ast()).
+		 */
+		tf->tf_scratch.gr10 = error;
+		if (error == 0) {
+			tf->tf_scratch.gr8 = td->td_retval[0];
+			tf->tf_scratch.gr9 = td->td_retval[1];
+		} else if (error != ERESTART) {
+			if (error < p->p_sysent->sv_errsize)
 				error = p->p_sysent->sv_errtbl[error];
+			/*
+			 * Translated error codes are returned in r8. User
+			 * processes use the translated error code.
+			 */
+			tf->tf_scratch.gr8 = error;
 		}
-		framep->tf_r[FRAME_R8] = error;
-		framep->tf_r[FRAME_R10] = 1;
-		break;
 	}
 
 	/*
@@ -883,12 +1009,13 @@ syscall(int code, u_int64_t *args, struct trapframe *framep)
 	if ((callp->sy_narg & SYF_MPSAFE) == 0)
 		mtx_unlock(&Giant);
 
-	userret(td, framep, sticks);
+	userret(td, tf, sticks);
 
 #ifdef KTRACE
 	if (KTRPOINT(td, KTR_SYSRET))
 		ktrsysret(code, error, td->td_retval[0]);
 #endif
+
 	/*
 	 * This works because errno is findable through the
 	 * register set.  If we ever support an emulation where this
@@ -899,10 +1026,13 @@ syscall(int code, u_int64_t *args, struct trapframe *framep)
 #ifdef DIAGNOSTIC
 	cred_free_thread(td);
 #endif
+
 	WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
 	    (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???");
 	mtx_assert(&sched_lock, MA_NOTOWNED);
 	mtx_assert(&Giant, MA_NOTOWNED);
+
+	return (error);
 }
 
 #include <i386/include/psl.h>
@@ -933,9 +1063,9 @@ ia32_syscall(struct trapframe *framep)
 	td->td_frame = framep;
 	if (td->td_ucred != p->p_ucred) 
 		cred_update_thread(td);
-	params = (caddr_t)(framep->tf_r[FRAME_SP] & ((1L<<32)-1))
-		+ sizeof(u_int32_t);
-	code = framep->tf_r[FRAME_R8]; /* eax */
+	params = (caddr_t)(framep->tf_special.sp & ((1L<<32)-1))
+	    + sizeof(u_int32_t);
+	code = framep->tf_scratch.gr8;		/* eax */
 	orig_eflags = ia64_get_eflag();
 
 	if (p->p_sysent->sv_prepsyscall) {
@@ -1001,7 +1131,7 @@ ia32_syscall(struct trapframe *framep)
 
 	if (error == 0) {
 		td->td_retval[0] = 0;
-		td->td_retval[1] = framep->tf_r[FRAME_R10]; /* edx */
+		td->td_retval[1] = framep->tf_scratch.gr10;	/* edx */
 
 		STOPEVENT(p, S_SCE, narg);
 
@@ -1010,8 +1140,8 @@ ia32_syscall(struct trapframe *framep)
 
 	switch (error) {
 	case 0:
-		framep->tf_r[FRAME_R8] = td->td_retval[0]; /* eax */
-		framep->tf_r[FRAME_R10] = td->td_retval[1]; /* edx */
+		framep->tf_scratch.gr8 = td->td_retval[0];	/* eax */
+		framep->tf_scratch.gr10 = td->td_retval[1];	/* edx */
 		ia64_set_eflag(ia64_get_eflag() & ~PSL_C);
 		break;
 
@@ -1020,7 +1150,7 @@ ia32_syscall(struct trapframe *framep)
 		 * Reconstruct pc, assuming lcall $X,y is 7 bytes,
 		 * int 0x80 is 2 bytes. XXX Assume int 0x80.
 		 */
-		framep->tf_cr_iip -= 2;
+		framep->tf_special.iip -= 2;
 		break;
 
 	case EJUSTRETURN:
@@ -1033,7 +1163,7 @@ ia32_syscall(struct trapframe *framep)
    			else
   				error = p->p_sysent->sv_errtbl[error];
 		}
-		framep->tf_r[FRAME_R8] = error;
+		framep->tf_scratch.gr8 = error;
 		ia64_set_eflag(ia64_get_eflag() | PSL_C);
 		break;
 	}
diff --git a/sys/ia64/ia64/unaligned.c b/sys/ia64/ia64/unaligned.c
index 78981be..0ebc2db 100644
--- a/sys/ia64/ia64/unaligned.c
+++ b/sys/ia64/ia64/unaligned.c
@@ -34,7 +34,6 @@
 #include <vm/vm_extern.h>
 #include <machine/frame.h>
 #include <machine/inst.h>
-#include <machine/rse.h>
 
 #define sign_extend(imm, w) (((int64_t)(imm) << (64 - (w))) >> (64 - (w)))
 
@@ -153,16 +152,45 @@ static int
 read_register(struct trapframe *framep, struct thread *td,
 	      int reg, u_int64_t *valuep)
 {
-	if (reg == 0) {
-		*valuep = 0; 
-		return 0;
-	} else if (reg < 32) {
-		*valuep = framep->tf_r[reg - 1];
-		return 0;
+
+	if (reg < 32) {
+		switch (reg) {
+		case 0:  *valuep = 0; break;
+		case 1:  *valuep = framep->tf_special.gp; break;
+		case 2:	 *valuep = framep->tf_scratch.gr2; break;
+		case 3:  *valuep = framep->tf_scratch.gr3; break;
+		case 8:  *valuep = framep->tf_scratch.gr8; break;
+		case 9:  *valuep = framep->tf_scratch.gr9; break;
+		case 10: *valuep = framep->tf_scratch.gr10; break;
+		case 11: *valuep = framep->tf_scratch.gr11; break;
+		case 12: *valuep = framep->tf_special.sp; break;
+		case 13: *valuep = framep->tf_special.tp; break;
+		case 14: *valuep = framep->tf_scratch.gr14; break;
+		case 15: *valuep = framep->tf_scratch.gr15; break;
+		case 16: *valuep = framep->tf_scratch.gr16; break;
+		case 17: *valuep = framep->tf_scratch.gr17; break;
+		case 18: *valuep = framep->tf_scratch.gr18; break;
+		case 19: *valuep = framep->tf_scratch.gr19; break;
+		case 20: *valuep = framep->tf_scratch.gr20; break;
+		case 21: *valuep = framep->tf_scratch.gr21; break;
+		case 22: *valuep = framep->tf_scratch.gr22; break;
+		case 23: *valuep = framep->tf_scratch.gr23; break;
+		case 24: *valuep = framep->tf_scratch.gr24; break;
+		case 25: *valuep = framep->tf_scratch.gr25; break;
+		case 26: *valuep = framep->tf_scratch.gr26; break;
+		case 27: *valuep = framep->tf_scratch.gr27; break;
+		case 28: *valuep = framep->tf_scratch.gr28; break;
+		case 29: *valuep = framep->tf_scratch.gr29; break;
+		case 30: *valuep = framep->tf_scratch.gr30; break;
+		case 31: *valuep = framep->tf_scratch.gr31; break;
+		default:
+			return (EINVAL);
+		}
 	} else {
-		u_int64_t cfm = framep->tf_cr_ifs;
-		u_int64_t *bsp = (u_int64_t *) (td->td_kstack
-						+ framep->tf_ndirty);
+#if 0
+		u_int64_t cfm = framep->tf_special.cfm;
+		u_int64_t *bsp = (u_int64_t *)(td->td_kstack +
+		    framep->tf_ndirty);
 		int sof = cfm & 0x7f;
 		int sor = 8*((cfm >> 14) & 15);
 		int rrb_gr = (cfm >> 18) & 0x7f;
@@ -182,23 +210,54 @@ read_register(struct trapframe *framep, struct thread *td,
 		}
 
 		*valuep = *ia64_rse_register_address(bsp, reg);
-		return 0;
+		return (0);
+#else
+		return (EINVAL);
+#endif
 	}
-
-	return EINVAL;
+	return (0);
 }
 
 static int
 write_register(struct trapframe *framep, struct thread *td,
 	      int reg, u_int64_t value)
 {
-	if (reg == 0) {
-		return EINVAL;	/* can't happen */
-	} else if (reg < 32) {
-		framep->tf_r[reg - 1] = value;
-		return 0;
+
+	if (reg < 32) {
+		switch (reg) {
+		case 1:  framep->tf_special.gp = value; break;
+		case 2:	 framep->tf_scratch.gr2 = value; break;
+		case 3:  framep->tf_scratch.gr3 = value; break;
+		case 8:  framep->tf_scratch.gr8 = value; break;
+		case 9:  framep->tf_scratch.gr9 = value; break;
+		case 10: framep->tf_scratch.gr10 = value; break;
+		case 11: framep->tf_scratch.gr11 = value; break;
+		case 12: framep->tf_special.sp = value; break;
+		case 13: framep->tf_special.tp = value; break;
+		case 14: framep->tf_scratch.gr14 = value; break;
+		case 15: framep->tf_scratch.gr15 = value; break;
+		case 16: framep->tf_scratch.gr16 = value; break;
+		case 17: framep->tf_scratch.gr17 = value; break;
+		case 18: framep->tf_scratch.gr18 = value; break;
+		case 19: framep->tf_scratch.gr19 = value; break;
+		case 20: framep->tf_scratch.gr20 = value; break;
+		case 21: framep->tf_scratch.gr21 = value; break;
+		case 22: framep->tf_scratch.gr22 = value; break;
+		case 23: framep->tf_scratch.gr23 = value; break;
+		case 24: framep->tf_scratch.gr24 = value; break;
+		case 25: framep->tf_scratch.gr25 = value; break;
+		case 26: framep->tf_scratch.gr26 = value; break;
+		case 27: framep->tf_scratch.gr27 = value; break;
+		case 28: framep->tf_scratch.gr28 = value; break;
+		case 29: framep->tf_scratch.gr29 = value; break;
+		case 30: framep->tf_scratch.gr30 = value; break;
+		case 31: framep->tf_scratch.gr31 = value; break;
+		default:
+			return (EINVAL);
+		}
 	} else {
-		u_int64_t cfm = framep->tf_cr_ifs;
+#if 0
+		u_int64_t cfm = framep->tf_special.cfm;
 		u_int64_t *bsp = (u_int64_t *) (td->td_kstack
 						+ framep->tf_ndirty);
 		int sof = cfm & 0x7f;
@@ -221,9 +280,11 @@ write_register(struct trapframe *framep, struct thread *td,
 
 		*ia64_rse_register_address(bsp, reg) = value;
 		return 0;
+#else
+		return (EINVAL);
+#endif
 	}
-
-	return EINVAL;
+	return (0);
 }
 
 /*
@@ -367,7 +428,7 @@ invala_e(int reg)
 int
 unaligned_fixup(struct trapframe *framep, struct thread *td)
 {
-	vm_offset_t va = framep->tf_cr_ifa;
+	vm_offset_t va = framep->tf_special.ifa;
 	int doprint, dofix, dosigbus;
 	int signal, size = 0;
 	unsigned long uac;
@@ -399,7 +460,7 @@ unaligned_fixup(struct trapframe *framep, struct thread *td)
 	 * If psr.ac is set, then clearly the user program *wants* to
 	 * fault.
 	 */
-	if (framep->tf_cr_ipsr & IA64_PSR_AC) {
+	if (framep->tf_special.psr & IA64_PSR_AC) {
 		dofix = 0;
 		dosigbus = 1;
 	}
@@ -419,10 +480,10 @@ unaligned_fixup(struct trapframe *framep, struct thread *td)
 	 * offending instruction.
 	 * XXX assume that the instruction is in an 'M' slot.
 	 */
-	copyin((const void *) framep->tf_cr_iip, &low, 8);
-	copyin((const void *) (framep->tf_cr_iip + 8), &high, 8);
+	copyin((const void *) framep->tf_special.iip, &low, 8);
+	copyin((const void *) (framep->tf_special.iip + 8), &high, 8);
 	ia64_unpack_bundle(low, high, &b);
-	slot = (framep->tf_cr_ipsr >> 41) & 3;
+	slot = (framep->tf_special.psr >> 41) & 3;
 	ins.ins = b.slot[slot];
 
 	decoded = 0;
@@ -451,7 +512,7 @@ unaligned_fixup(struct trapframe *framep, struct thread *td)
 	 */
 	if (doprint) {
 		uprintf("pid %d (%s): unaligned access: va=0x%lx pc=0x%lx",
-			p->p_pid, p->p_comm, va, framep->tf_cr_iip);
+			p->p_pid, p->p_comm, va, framep->tf_special.iip);
 		if (decoded) {
 			uprintf(" op=");
 			if (dec.isload) {
@@ -500,7 +561,7 @@ unaligned_fixup(struct trapframe *framep, struct thread *td)
 		 */
 		__asm __volatile("flushrs");
 
-		isr = framep->tf_cr_isr;
+		isr = framep->tf_special.isr;
 		error = read_register(framep, td, dec.basereg, &addr);
 		if (error) {
 			signal = SIGBUS;
@@ -565,12 +626,12 @@ unaligned_fixup(struct trapframe *framep, struct thread *td)
 			 * Advance to the instruction following the
 			 * one which faulted.
 			 */
-			if ((framep->tf_cr_ipsr & IA64_PSR_RI)
+			if ((framep->tf_special.psr & IA64_PSR_RI)
 			    == IA64_PSR_RI_2) {
-				framep->tf_cr_ipsr &= ~IA64_PSR_RI;
-				framep->tf_cr_iip += 16;
+				framep->tf_special.psr &= ~IA64_PSR_RI;
+				framep->tf_special.iip += 16;
 			} else {
-				framep->tf_cr_ipsr += IA64_PSR_RI_1;
+				framep->tf_special.psr += IA64_PSR_RI_1;
 			}
 		}
 	} else {
diff --git a/sys/ia64/ia64/unwind.c b/sys/ia64/ia64/unwind.c
index 2da7041..5f1c3e3 100644
--- a/sys/ia64/ia64/unwind.c
+++ b/sys/ia64/ia64/unwind.c
@@ -34,1479 +34,274 @@
 
 #include <machine/frame.h>
 #include <machine/unwind.h>
-#include <machine/rse.h>
 
-#ifdef UNWIND_DEBUG
-#define DPF(x)	printf x
-#else
-#define DPF(x)
-#endif
-
-MALLOC_DEFINE(M_UNWIND, "Unwind table", "Unwind table information");
+#include <uwx.h>
 
-struct ia64_unwind_table_entry {
-	u_int64_t	ue_start;	/* procedure start */
-	u_int64_t	ue_end;		/* procedure end */
-	u_int64_t	ue_info;	/* offset to procedure descriptors */
-};
+MALLOC_DEFINE(M_UNWIND, "Unwind", "Unwind information");
 
-struct ia64_unwind_info {
-	u_int64_t	ui_length	: 32; /* length in 64bit units */
-	u_int64_t	ui_flags	: 16;
-	u_int64_t	ui_version	: 16;
+struct unw_entry {
+	uint64_t	ue_start;	/* procedure start */
+	uint64_t	ue_end;		/* procedure end */
+	uint64_t	ue_info;	/* offset to procedure descriptors */
 };
 
-LIST_HEAD(ia64_unwind_table_list, ia64_unwind_table);
-
-struct ia64_unwind_table {
-	LIST_ENTRY(ia64_unwind_table) ut_link;
-	u_int64_t	ut_base;
-	u_int64_t	ut_limit;
-	struct ia64_unwind_table_entry *ut_start;
-	struct ia64_unwind_table_entry *ut_end;
+struct unw_table {
+	LIST_ENTRY(unw_table) ut_link;
+	uint64_t	ut_base;
+	uint64_t	ut_limit;
+	struct unw_entry *ut_start;
+	struct unw_entry *ut_end;
 };
 
-struct unwind_reg {
-	u_int64_t	ur_value;	/* current value */
-	u_int64_t	*ur_save;	/* save location */
-	int		ur_when;	/* when the save happened */
-};
-
-struct unwind_fpreg {
-	struct ia64_fpreg ur_value;	/* current value */
-	struct ia64_fpreg *ur_save;	/* save location */
-	int		ur_when;	/* when the save happened */
-};
+LIST_HEAD(unw_table_list, unw_table);
 
-struct ia64_unwind_state {
-	LIST_ENTRY(ia64_unwind_state) us_link; /* free list */
+static struct unw_table_list unw_tables;
 
-	/*
-	 * Current register state and location of saved register state
-	 */
-	struct register_state {
-		struct unwind_reg rs_psp;
-		struct unwind_reg rs_pfs;
-		struct unwind_reg rs_preds;
-		struct unwind_reg rs_unat;
-		struct unwind_reg rs_lc;
-		struct unwind_reg rs_rnat;
-		struct unwind_reg rs_bsp;
-		struct unwind_reg rs_bspstore;
-		struct unwind_reg rs_fpsr;
-		struct unwind_reg rs_priunat;
-		struct unwind_reg rs_br[8];
-		struct unwind_reg rs_gr[32];
-		struct unwind_fpreg rs_fr[32];
-		u_int64_t	rs_stack_size;
-	}		us_regs;
-
-	/*
-	 * Variables used while parsing unwind records.
-	 */
-	u_int64_t	us_ip;		/* value of IP for this frame */
-	int		us_ri;		/* RI field from cr.ipsr */
-	u_int64_t	us_pc;		/* slot offset in procedure */
-	u_int64_t	*us_bsp;	/* backing store for frame */
-	u_int64_t	us_cfm;		/* CFM value for frame */
-	u_int64_t	*us_spill;	/* spill_base location */
-	int		us_spilloff;	/* offset into spill area */
-	int		us_grmask;	/* mask of grs being spilled */
-	int		us_frmask;	/* mask of frs being spilled */
-	int		us_brmask;	/* mask of brs being spilled */
-};
-
-static int ia64_unwind_initialised;
-static struct ia64_unwind_table_list ia64_unwind_tables;
-#define MAX_UNWIND_STATES 4
-static struct ia64_unwind_state ia64_unwind_state_static[MAX_UNWIND_STATES];
-static LIST_HEAD(ia64_unwind_state_list, ia64_unwind_state) ia64_unwind_states;
-
-static void
-ia64_initialise_unwind(void *arg __unused)
+static void *
+unw_alloc(size_t sz)
 {
-	int i;
-
-	KASSERT(!ia64_unwind_initialised, ("foo"));
 
-	LIST_INIT(&ia64_unwind_tables);
-	LIST_INIT(&ia64_unwind_states);
-	for (i = 0; i < MAX_UNWIND_STATES; i++) {
-		LIST_INSERT_HEAD(&ia64_unwind_states,
-		    &ia64_unwind_state_static[i], us_link);
-	}
-
-	ia64_unwind_initialised = 1;
+	return (malloc(sz, M_UNWIND, M_WAITOK));
 }
-SYSINIT(unwind, SI_SUB_KMEM, SI_ORDER_ANY, ia64_initialise_unwind, 0);
 
-static struct ia64_unwind_table *
-find_table(u_int64_t ip)
+static void
+unw_free(void *p)
 {
-	struct ia64_unwind_table *ut;
 
-	LIST_FOREACH(ut, &ia64_unwind_tables, ut_link) {
-		if (ip >= ut->ut_base && ip < ut->ut_limit)
-			return ut;
-	}
-	return 0;
+	free(p, M_UNWIND);
 }
 
-static struct ia64_unwind_table_entry *
-find_entry(struct ia64_unwind_table *ut, u_int64_t ip)
+#if 0
+static struct unw_entry *
+unw_entry_lookup(struct unw_table *ut, uint64_t ip)
 {
-	struct ia64_unwind_table_entry *start;
-	struct ia64_unwind_table_entry *end;
-	struct ia64_unwind_table_entry *mid;
+	struct unw_entry *end, *mid, *start;
 
 	ip -= ut->ut_base;
 	start = ut->ut_start;
 	end = ut->ut_end - 1;
 	while (start < end) {
-		mid = start + (end - start) / 2;
-		if (ip < mid->ue_start) {
-			if (end == mid)
-				break;
+		mid = start + ((end - start) >> 1);
+		if (ip < mid->ue_start)
 			end = mid;
-		} else if (ip >= mid->ue_end) {
-			if (start == mid)
-				break;
-			start = mid;
-		} else
-			return mid;
-	}
-
-	return 0;
-}
-
-int
-ia64_add_unwind_table(vm_offset_t base, vm_offset_t start, vm_offset_t end)
-{
-	struct ia64_unwind_table *ut;
-
-	KASSERT(ia64_unwind_initialised, ("foo"));
-
-	ut = malloc(sizeof(struct ia64_unwind_table), M_UNWIND, M_NOWAIT);
-	if (ut == NULL)
-		return (ENOMEM);
-
-	ut->ut_base = base;
-	ut->ut_start = (struct ia64_unwind_table_entry*)start;
-	ut->ut_end = (struct ia64_unwind_table_entry*)end;
-	ut->ut_limit = base + ut->ut_end[-1].ue_end;
-	LIST_INSERT_HEAD(&ia64_unwind_tables, ut, ut_link);
-
-	if (bootverbose)
-		printf("UNWIND: table added: base=%lx, start=%lx, end=%lx\n",
-		    base, start, end);
-
-	return (0);
-}
-
-void
-ia64_delete_unwind_table(vm_offset_t base)
-{
-	struct ia64_unwind_table *ut;
-
-	KASSERT(ia64_unwind_initialised, ("foo"));
-
-	ut = find_table(base);
-	if (ut != NULL) {
-		LIST_REMOVE(ut, ut_link);
-		free(ut, M_UNWIND);
-		if (bootverbose)
-			printf("UNWIND: table removed: base=%lx\n", base);
-	}
-}
-
-struct ia64_unwind_state *
-ia64_create_unwind_state(struct trapframe *framep)
-{
-	struct ia64_unwind_state *us;
-	int i;
-
-	if (!ia64_unwind_initialised)
-		return 0;
-
-	us = LIST_FIRST(&ia64_unwind_states);
-	if (us) {
-		LIST_REMOVE(us, us_link);
-	} else {
-		us = malloc(sizeof(struct ia64_unwind_state),
-			    M_UNWIND, M_NOWAIT);
-		if (!us)
-			return 0;
-	}
-
-	bzero(us, sizeof(*us));
-	us->us_regs.rs_psp.ur_value = framep->tf_r[FRAME_SP];
-	us->us_regs.rs_pfs.ur_value = framep->tf_ar_pfs;
-	us->us_regs.rs_preds.ur_value = framep->tf_pr;
-	us->us_regs.rs_unat.ur_value = framep->tf_ar_unat;
-	us->us_regs.rs_rnat.ur_value = framep->tf_ar_rnat;
-	us->us_regs.rs_bsp.ur_value =
-		(u_int64_t) (framep->tf_ar_bspstore + framep->tf_ndirty);
-	us->us_regs.rs_bspstore.ur_value = framep->tf_ar_bspstore;
-	us->us_regs.rs_fpsr.ur_value = framep->tf_ar_fpsr;
-	for (i = 0; i < 8; i++) {
-		us->us_regs.rs_br[i].ur_value = framep->tf_b[i];
-	}
-	us->us_regs.rs_gr[0].ur_value = 0;
-	for (i = 1; i < 32; i++) {
-		us->us_regs.rs_gr[i].ur_value = framep->tf_r[i-1];
-	}
-	for (i = 6; i < 16; i++) {
-		us->us_regs.rs_fr[i].ur_value = framep->tf_f[i-6];
+		else if (ip >= mid->ue_end)
+			start = mid + 1;
+		else
+			break;
 	}
-
-	us->us_ip = framep->tf_cr_iip;
-	us->us_ri = (framep->tf_cr_ipsr & IA64_PSR_RI) >> 41;
-	us->us_spill = (u_int64_t *) us->us_regs.rs_gr[12].ur_value;
-	us->us_cfm = framep->tf_cr_ifs;
-	us->us_bsp = ia64_rse_previous_frame
-		((u_int64_t *) us->us_regs.rs_bsp.ur_value, us->us_cfm & 0x7f);
-
-	return us;
-}
-
-void
-ia64_free_unwind_state(struct ia64_unwind_state *us)
-{
-	LIST_INSERT_HEAD(&ia64_unwind_states, us, us_link);
-}
-
-u_int64_t
-ia64_unwind_state_get_ip(struct ia64_unwind_state *us)
-{
-	return us->us_ip + us->us_ri;
-}
-
-u_int64_t
-ia64_unwind_state_get_sp(struct ia64_unwind_state *us)
-{
-	return us->us_regs.rs_gr[12].ur_value;
-}
-
-u_int64_t
-ia64_unwind_state_get_cfm(struct ia64_unwind_state *us)
-{
-	return us->us_cfm;
-}
-
-u_int64_t *
-ia64_unwind_state_get_bsp(struct ia64_unwind_state *us)
-{
-	return us->us_bsp;
-}
-
-static u_int64_t
-read_uleb128(u_int8_t **pp)
-{
-	u_int8_t *p = *pp;
-	u_int8_t b;
-	u_int64_t res;
-
-	res = 0;
-	do {
-		b = *p++;
-		res = (res << 7) | (b & 0x7f);
-	} while (b & (1 << 7));
-
-	*pp = p;
-	return res;
-}
-
-#define PROCESS_WHEN(us, reg, t)				\
-do {								\
-	DPF(("register %s was saved at offset %d\n",		\
-	     #reg, t));						\
-	us->us_regs.rs_##reg.ur_when = t;			\
-} while (0)
-
-#define PROCESS_GR(us, reg, gr)					\
-do {								\
-	DPF(("save location for %s at r%d\n", #reg, gr));	\
-	us->us_regs.rs_##reg.ur_save = find_gr(us, gr);		\
-} while (0)							\
-
-#define PROCESS_BR(us, reg, br)					\
-do {								\
-	DPF(("save location for %s at b%d\n", #reg, br));	\
-	us->us_regs.rs_##reg.ur_save =				\
-		&us->us_regs.rs_br[br].ur_value;		\
-} while (0)
-
-#define PROCESS_GRMEM(us, reg)					\
-do {								\
-	DPF(("save location for %s at spill+%d\n",		\
-	     #reg, us->us_spilloff));				\
-	us->us_regs.rs_##reg.ur_save =				\
-		&us->us_spill[us->us_spilloff];			\
-	us->us_spilloff += 8;					\
-} while (0)
-
-#define PROCESS_FRMEM(us, reg)					\
-do {								\
-	DPF(("save location for %s at spill+%d\n",		\
-	     #reg, us->us_spilloff));				\
-	us->us_regs.rs_##reg.ur_save =				\
-		(struct ia64_fpreg *)				\
-		&us->us_spill[us->us_spilloff];			\
-	us->us_spilloff += 16;					\
-} while (0)
-
-#define PROCESS_SPREL(us, reg, spoff)				\
-do {								\
-	DPF(("save location for %s at sp+%d\n",			\
-	     #reg, 4*spoff));					\
-	us->us_regs.rs_##reg.ur_save = (u_int64_t *)		\
-		(us->us_regs.rs_gr[12].ur_value + 4*spoff);	\
-} while (0)
-
-#define PROCESS_SPREL_WHEN(us, reg, spoff, t)			\
-do {								\
-	PROCESS_SPREL(us, reg, spoff);				\
-	PROCESS_WHEN(us, reg, t);				\
-} while (0)
-
-#define PROCESS_PSPREL(us, reg, pspoff)				\
-do {								\
-	DPF(("save location for %s at psp+%d\n",		\
-	     #reg, 16-4*pspoff));				\
-	us->us_regs.rs_##reg.ur_save = (u_int64_t *)		\
-		(us->us_regs.rs_psp.ur_value + 16-4*pspoff);	\
-} while (0)
-
-#define PROCESS_PSPREL_WHEN(us, reg, pspoff, t)			\
-do {								\
-	PROCESS_PSPREL(us, reg, pspoff);			\
-	PROCESS_WHEN(us, reg, t);				\
-} while (0)
-
-static u_int64_t *
-find_gr(struct ia64_unwind_state *us, int gr)
-{
-	if (gr < 32)
-		return &us->us_regs.rs_gr[gr].ur_value;
-	else
-		return ia64_rse_register_address(us->us_bsp, gr);
+	return ((start < end) ? mid : NULL);
 }
+#endif
 
-static void
-parse_prologue(struct ia64_unwind_state *us, int rlen)
+static struct unw_table *
+unw_table_lookup(uint64_t ip)
 {
-}
+	struct unw_table *ut;
 
-static void
-parse_prologue_gr(struct ia64_unwind_state *us, int rlen,
-		  int mask, int grsave)
-{
-	if (mask & 8) {
-		PROCESS_GR(us, br[0], grsave);
-		grsave++;
-	}
-	if (mask & 4) {
-		PROCESS_GR(us, pfs, grsave);
-		grsave++;
-	}
-	if (mask & 2) {
-		PROCESS_GR(us, psp, grsave);
-		grsave++;
-	}
-	if (mask & 1) {
-		PROCESS_GR(us, preds, grsave);
-		grsave++;
+	LIST_FOREACH(ut, &unw_tables, ut_link) {
+		if (ip >= ut->ut_base && ip < ut->ut_limit)
+			return (ut);
 	}
+	return (NULL);
 }
 
-static void
-parse_mem_stack_f(struct ia64_unwind_state *us, int t, int size)
-{
-	DPF(("restore value for psp is sp+%d at offset %d\n",
-	     16*size, t));
-	us->us_regs.rs_psp.ur_when = t;
-	us->us_regs.rs_stack_size = 16*size;
-}
-
-static void
-parse_mem_stack_v(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, psp, t);
-}
-
-static void
-parse_psp_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, psp, gr);
-}
-
-static void
-parse_psp_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, psp, spoff);
-}
-
-static void
-parse_rp_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, br[0], t);
-}
-
-static void
-parse_rp_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, br[0], gr);
-}
-
-static void
-parse_rp_br(struct ia64_unwind_state *us, int br)
-{
-	PROCESS_BR(us, br[0], br);
-}
-
-static void
-parse_rp_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, br[0], pspoff);
-}
-
-static void
-parse_rp_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, br[0], spoff);
-}
-
-static void
-parse_pfs_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, pfs, t);
-}
-
-static void
-parse_pfs_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, pfs, gr);
-}
-
-static void
-parse_pfs_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, pfs, pspoff);
-}
-
-static void
-parse_pfs_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, pfs, spoff);
-}
-
-static void
-parse_preds_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, preds, t);
-}
-
-static void
-parse_preds_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, preds, gr);
-}
-
-static void
-parse_preds_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, preds, pspoff);
-}
-
-static void
-parse_preds_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, preds, spoff);
-}
-
-static void
-parse_fr_mem(struct ia64_unwind_state *us, int frmask)
-{
-	us->us_frmask = frmask;
-}
-
-static void
-parse_frgr_mem(struct ia64_unwind_state *us, int grmask, int frmask)
+static int
+unw_cb_copyin(int req, char *to, uint64_t from, int len, intptr_t tok)
 {
-	us->us_grmask = grmask;
-	if (grmask & 1)
-		PROCESS_GRMEM(us, gr[4]);
-	if (grmask & 2)
-		PROCESS_GRMEM(us, gr[5]);
-	if (grmask & 4)
-		PROCESS_GRMEM(us, gr[6]);
-	if (grmask & 8)
-		PROCESS_GRMEM(us, gr[7]);
+	struct unw_regstate *rs = (void*)tok;
+	int reg;
 
-	us->us_frmask = frmask;
-	if (frmask & 1)
-		PROCESS_FRMEM(us, fr[2]);
-	if (frmask & 2)
-		PROCESS_FRMEM(us, fr[3]);
-	if (frmask & 4)
-		PROCESS_FRMEM(us, fr[4]);
-	if (frmask & 8)
-		PROCESS_FRMEM(us, fr[5]);
-	if (frmask & 16)
-		PROCESS_FRMEM(us, fr[16]);
-	if (frmask & 32)
-		PROCESS_FRMEM(us, fr[17]);
-	if (frmask & 64)
-		PROCESS_FRMEM(us, fr[18]);
-	if (frmask & 128)
-		PROCESS_FRMEM(us, fr[19]);
-	if (frmask & 256)
-		PROCESS_FRMEM(us, fr[20]);
-	if (frmask & 512)
-		PROCESS_FRMEM(us, fr[21]);
-	if (frmask & 1024)
-		PROCESS_FRMEM(us, fr[22]);
-	if (frmask & 2048)
-		PROCESS_FRMEM(us, fr[24]);
-	if (frmask & 4096)
-		PROCESS_FRMEM(us, fr[25]);
-	if (frmask & 8192)
-		PROCESS_FRMEM(us, fr[26]);
-	if (frmask & 16384)
-		PROCESS_FRMEM(us, fr[27]);
-	if (frmask & 32768)
-		PROCESS_FRMEM(us, fr[28]);
-	if (frmask & 65536)
-		PROCESS_FRMEM(us, fr[29]);
-	if (frmask & 131072)
-		PROCESS_FRMEM(us, fr[30]);
-	if (frmask & 262144)
-		PROCESS_FRMEM(us, fr[31]);
-}
+	switch (req) {
+	case UWX_COPYIN_UINFO:
+		break;
+	case UWX_COPYIN_MSTACK:
+		*((uint64_t*)to) = *((uint64_t*)from);
+		return (8);
+	case UWX_COPYIN_RSTACK:
+		*((uint64_t*)to) = *((uint64_t*)from);
+		return (8);
+	case UWX_COPYIN_REG:
+		if (from == UWX_REG_PFS)
+			from = rs->frame->tf_special.pfs;
+		else if (from == UWX_REG_PREDS)
+			from = rs->frame->tf_special.pr;
+		else if (from == UWX_REG_RNAT)
+			from = rs->frame->tf_special.rnat;
+		else if (from == UWX_REG_UNAT)
+			from = rs->frame->tf_special.unat;
+		else if (from >= UWX_REG_GR(0) && from <= UWX_REG_GR(127)) {
+			reg = from - UWX_REG_GR(0);
+			if (reg == 1)
+				from = rs->frame->tf_special.gp;
+			else if (reg == 12)
+				from = rs->frame->tf_special.sp;
+			else if (reg == 13)
+				from = rs->frame->tf_special.tp;
+			else if (reg >= 2 && reg <= 3)
+				from = (&rs->frame->tf_scratch.gr2)[reg - 2];
+			else if (reg >= 8 && reg <= 11)
+				from = (&rs->frame->tf_scratch.gr8)[reg - 8];
+			else if (reg >= 14 && reg <= 31)
+				from = (&rs->frame->tf_scratch.gr14)[reg - 14];
+			else
+				goto oops;
+		} else if (from >= UWX_REG_BR(0) && from <= UWX_REG_BR(7)) {
+			reg = from - UWX_REG_BR(0);
+			if (reg == 0)
+				from = rs->frame->tf_special.rp;
+			else if (reg >= 6 && reg <= 7)
+				from = (&rs->frame->tf_scratch.br6)[reg - 6];
+			else
+				goto oops;
+		} else
+			goto oops;
 
-static void
-parse_gr_gr(struct ia64_unwind_state *us, int grmask, int gr)
-{
-	us->us_grmask = grmask;
-	if (grmask & 1) {
-		PROCESS_GR(us, gr[4], gr);
-		gr++;
+		*((uint64_t*)to) = from;
+		return (len);
 	}
-	if (grmask & 2) {
-		PROCESS_GR(us, gr[5], gr);
-		gr++;
-	}
-	if (grmask & 4) {
-		PROCESS_GR(us, gr[6], gr);
-		gr++;
-	}
-	if (grmask & 8) {
-		PROCESS_GR(us, gr[7], gr);
-		gr++;
-	}
-}
 
-static void
-parse_gr_mem(struct ia64_unwind_state *us, int grmask)
-{
-	us->us_grmask = grmask;
-	if (grmask & 1)
-		PROCESS_GRMEM(us, gr[4]);
-	if (grmask & 2)
-		PROCESS_GRMEM(us, gr[5]);
-	if (grmask & 4)
-		PROCESS_GRMEM(us, gr[6]);
-	if (grmask & 8)
-		PROCESS_GRMEM(us, gr[7]);
-}
-
-static void
-parse_br_mem(struct ia64_unwind_state *us, int brmask)
-{
-	us->us_brmask = brmask;
-	if (brmask & 1)
-		PROCESS_GRMEM(us, br[1]);
-	if (brmask & 2)
-		PROCESS_GRMEM(us, br[2]);
-	if (brmask & 4)
-		PROCESS_GRMEM(us, br[3]);
-	if (brmask & 8)
-		PROCESS_GRMEM(us, br[4]);
-	if (brmask & 16)
-		PROCESS_GRMEM(us, br[5]);
-}
+ oops:
+	printf("UNW: %s(%d, %p, %lx, %d, %lx)\n", __func__, req, to, from,
+	    len, tok);
 
-static void
-parse_br_gr(struct ia64_unwind_state *us, int brmask, int gr)
-{
-	us->us_brmask = brmask;
-	if (brmask & 1) {
-		PROCESS_GR(us, br[1], gr);
-		gr++;
-	}
-	if (brmask & 2) {
-		PROCESS_GR(us, br[2], gr);
-		gr++;
-	}
-	if (brmask & 4) {
-		PROCESS_GR(us, br[3], gr);
-		gr++;
-	}
-	if (brmask & 8) {
-		PROCESS_GR(us, br[4], gr);
-		gr++;
-	}
-	if (brmask & 16) {
-		PROCESS_GR(us, br[5], gr);
-		gr++;
-	}
-}
-
-static void
-parse_spill_base(struct ia64_unwind_state *us, int pspoff)
-{
-	DPF(("base of spill area at psp+%d\n", 16 - 4*pspoff));
-	us->us_spill = (u_int64_t *)
-		(us->us_regs.rs_psp.ur_value + 16 - 4*pspoff);
+	return (0);
 }
 
-static void
-parse_spill_mask(struct ia64_unwind_state *us, int rlen, u_int8_t *imask)
-{
-	int i, reg;
-	u_int8_t b;
-	static int frno[] = {
-		2, 3, 4, 5, 16, 17, 18, 19, 20, 21, 22,
-		23, 24, 25, 26, 27, 28, 29, 30, 31
-	};
-
-	for (i = 0; i < rlen; i++) {
-		b = imask[i / 4];
-		b = (b >> (2 * (3-(i & 3)))) & 3;
-		switch (b) {
-		case 0:
-			break;
-		case 1:
-			reg = frno[ffs(us->us_frmask) - 1];
-			DPF(("restoring fr[%d] at offset %d\n", reg, i));
-			us->us_regs.rs_fr[reg].ur_when = i;
-			break;
-		case 2:
-			reg = ffs(us->us_grmask) - 1 + 4;
-			DPF(("restoring gr[%d] at offset %d\n", reg, i));
-			us->us_regs.rs_gr[reg].ur_when = i;
-			break;
-		case 3:
-			reg = ffs(us->us_brmask) - 1 + 1;
-			DPF(("restoring br[%d] at offset %d\n", reg, i));
-			us->us_regs.rs_gr[reg].ur_when = i;
-			break;
-		}
+static int
+unw_cb_lookup(int req, uint64_t ip, intptr_t tok, uint64_t **vec)
+{
+	struct unw_regstate *rs = (void*)tok;
+	struct unw_table *ut;
+
+	switch (req) {
+	case UWX_LKUP_LOOKUP:
+		ut = unw_table_lookup(ip);
+		if (ut == NULL)
+			return (UWX_LKUP_NOTFOUND);
+		rs->keyval[0] = UWX_KEY_TBASE;
+		rs->keyval[1] = ut->ut_base;
+		rs->keyval[2] = UWX_KEY_USTART;
+		rs->keyval[3] = (intptr_t)ut->ut_start;
+		rs->keyval[4] = UWX_KEY_UEND;
+		rs->keyval[5] = (intptr_t)ut->ut_end;
+		rs->keyval[6] = 0;
+		rs->keyval[7] = 0;
+		*vec = rs->keyval;
+		return (UWX_LKUP_UTABLE);
+	case UWX_LKUP_FREE:
+		return (0);
 	}
-}
-
-static void
-parse_unat_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, unat, t);
-}
-
-static void
-parse_unat_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, unat, gr);
-}
-
-static void
-parse_unat_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, unat, pspoff);
-}
-
-static void
-parse_unat_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, unat, spoff);
-}
-
-static void
-parse_lc_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, lc, t);
-}
-
-static void
-parse_lc_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, lc, gr);
-}
-
-static void
-parse_lc_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, lc, pspoff);
-}
-
-static void
-parse_lc_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, lc, spoff);
-}
-
-static void
-parse_fpsr_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, fpsr, t);
-}
-
-static void
-parse_fpsr_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, fpsr, gr);
-}
-
-static void
-parse_fpsr_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, fpsr, pspoff);
-}
 
-static void
-parse_fpsr_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, fpsr, spoff);
-}
-
-static void
-parse_priunat_when_gr(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, priunat, t);
+	return (UWX_LKUP_ERR);
 }
 
-static void
-parse_priunat_when_mem(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, priunat, t);
-}
-
-static void
-parse_priunat_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, priunat, gr);
-}
-
-static void
-parse_priunat_psprel(struct ia64_unwind_state *us, int pspoff)
+int
+unw_create(struct unw_regstate *rs, struct trapframe *tf)
 {
-	PROCESS_PSPREL(us, priunat, pspoff);
-}
+	struct unw_table *ut;
+	uint64_t bsp;
+	int nats, sof, uwxerr;
 
-static void
-parse_priunat_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, priunat, spoff);
-}
+	ut = unw_table_lookup(tf->tf_special.iip);
+	if (ut == NULL)
+		return (ENOENT);
 
-static void
-parse_bsp_when(struct ia64_unwind_state *us, int t)
-{
-	PROCESS_WHEN(us, bsp, t);
-}
+	rs->frame = tf;
+	rs->env = uwx_init();
+	if (rs->env == NULL)
+		return (ENOMEM);
 
-static void
-parse_bsp_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, bsp, gr);
-}
+	uwxerr = uwx_register_callbacks(rs->env, (intptr_t)rs,
+	    unw_cb_copyin, unw_cb_lookup);
+	if (uwxerr)
+		return (EINVAL);		/* XXX */
 
-static void
-parse_bsp_psprel(struct ia64_unwind_state *us, int pspoff)
-{
-	PROCESS_PSPREL(us, bsp, pspoff);
-}
+	bsp = tf->tf_special.bspstore + tf->tf_special.ndirty;
+	sof = (int)(tf->tf_special.cfm & 0x7f);
+	nats = (sof + 63 - ((int)(bsp >> 3) & 0x3f)) / 63;
+	uwxerr = uwx_init_context(rs->env, tf->tf_special.iip,
+	    tf->tf_special.sp, bsp - ((sof + nats) << 3), tf->tf_special.cfm);
 
-static void
-parse_bsp_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, bsp, spoff);
+	return ((uwxerr) ? EINVAL : 0);		/* XXX */
 }
 
-static void
-parse_bspstore_when(struct ia64_unwind_state *us, int t)
+int
+unw_step(struct unw_regstate *rs)
 {
-	PROCESS_WHEN(us, bspstore, t);
-}
+	int uwxerr;
 
-static void
-parse_bspstore_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, bspstore, gr);
+	uwxerr = uwx_step(rs->env);
+	return ((uwxerr) ? EINVAL : 0);		/* XXX */
 }
 
-static void
-parse_bspstore_psprel(struct ia64_unwind_state *us, int pspoff)
+int
+unw_get_bsp(struct unw_regstate *s, uint64_t *r)
 {
-	PROCESS_PSPREL(us, bspstore, pspoff);
-}
+	int uwxerr;
 
-static void
-parse_bspstore_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, bspstore, spoff);
+	uwxerr = uwx_get_reg(s->env, UWX_REG_BSP, r);
+	return ((uwxerr) ? EINVAL : 0); 	/* XXX */
 }
 
-static void
-parse_rnat_when(struct ia64_unwind_state *us, int t)
+int
+unw_get_cfm(struct unw_regstate *s, uint64_t *r)
 {
-	PROCESS_WHEN(us, rnat, t);
-}
+	int uwxerr;
 
-static void
-parse_rnat_gr(struct ia64_unwind_state *us, int gr)
-{
-	PROCESS_GR(us, rnat, gr);
+	uwxerr = uwx_get_reg(s->env, UWX_REG_CFM, r);
+	return ((uwxerr) ? EINVAL : 0); 	/* XXX */
 }
 
-static void
-parse_rnat_psprel(struct ia64_unwind_state *us, int pspoff)
+int
+unw_get_ip(struct unw_regstate *s, uint64_t *r)
 {
-	PROCESS_PSPREL(us, rnat, pspoff);
-}
+	int uwxerr;
 
-static void
-parse_rnat_sprel(struct ia64_unwind_state *us, int spoff)
-{
-	PROCESS_SPREL(us, rnat, spoff);
+	uwxerr = uwx_get_reg(s->env, UWX_REG_IP, r);
+	return ((uwxerr) ? EINVAL : 0); 	/* XXX */
 }
 
-static void
-parse_epilogue(struct ia64_unwind_state *us, int t, int ecount)
+int
+unw_table_add(uint64_t base, uint64_t start, uint64_t end)
 {
-}
+	struct unw_table *ut;
 
-static void
-parse_label_state(struct ia64_unwind_state *us, int label)
-{
-}
+	ut = malloc(sizeof(struct unw_table), M_UNWIND, M_NOWAIT);
+	if (ut == NULL)
+		return (ENOMEM);
 
-static void
-parse_copy_state(struct ia64_unwind_state *us, int label)
-{
-}
+	ut->ut_base = base;
+	ut->ut_start = (struct unw_entry*)start;
+	ut->ut_end = (struct unw_entry*)end;
+	ut->ut_limit = base + ut->ut_end[-1].ue_end;
+	LIST_INSERT_HEAD(&unw_tables, ut, ut_link);
 
-static void
-parse_spill_psprel(struct ia64_unwind_state *us, int t,
-		   int reg, int pspoff)
-{
-	int type;
+	if (bootverbose)
+		printf("UNWIND: table added: base=%lx, start=%lx, end=%lx\n",
+		    base, start, end);
 
-	type = reg >> 5;
-	reg &= 0x1f;
-	switch (type) {
-	case 0:
-		DPF(("save location for gr[%d] at psp+%d at offset %d\n",
-		       reg, 16-4*pspoff, t));
-		us->us_regs.rs_gr[reg].ur_save = (u_int64_t *)
-			(us->us_regs.rs_psp.ur_value + 16-4*pspoff);
-		us->us_regs.rs_gr[reg].ur_when = t;
-		break;
-	case 1:
-		DPF(("save location for fr[%d] at psp+%d at offset %d\n",
-		       reg, 16-4*pspoff, t));
-		us->us_regs.rs_fr[reg].ur_save = (struct ia64_fpreg *)
-			(us->us_regs.rs_psp.ur_value + 16-4*pspoff);
-		us->us_regs.rs_fr[reg].ur_when = t;
-		break;
-	case 2:
-		DPF(("save location for br[%d] at psp+%d at offset %d\n",
-		       reg, 16-4*pspoff, t));
-		us->us_regs.rs_br[reg].ur_save = (u_int64_t *)
-			(us->us_regs.rs_psp.ur_value + 16-4*pspoff);
-		us->us_regs.rs_br[reg].ur_when = t;
-		break;
-	case 3:
-		switch (reg) {
-		case 0:
-			PROCESS_PSPREL_WHEN(us, preds, pspoff, t);
-			break;
-		case 1:
-			PROCESS_PSPREL_WHEN(us, psp, pspoff, t);
-			break;
-		case 2:
-			PROCESS_PSPREL_WHEN(us, priunat, pspoff, t);
-			break;
-		case 3:
-			PROCESS_PSPREL_WHEN(us, br[0], pspoff, t);
-			break;
-		case 4:
-			PROCESS_PSPREL_WHEN(us, bsp, pspoff, t);
-			break;
-		case 5:
-			PROCESS_PSPREL_WHEN(us, bspstore, pspoff, t);
-			break;
-		case 6:
-			PROCESS_PSPREL_WHEN(us, rnat, pspoff, t);
-			break;
-		case 7:
-			PROCESS_PSPREL_WHEN(us, unat, pspoff, t);
-			break;
-		case 8:
-			PROCESS_PSPREL_WHEN(us, fpsr, pspoff, t);
-			break;
-		case 9:
-			PROCESS_PSPREL_WHEN(us, pfs, pspoff, t);
-			break;
-		case 10:
-			PROCESS_PSPREL_WHEN(us, lc, pspoff, t);
-			break;
-		}
-	}
+	return (0);
 }
 
-static void
-parse_spill_sprel(struct ia64_unwind_state *us, int t,
-		  int reg, int spoff)
+void
+unw_table_remove(uint64_t base)
 {
-	int type;
+	struct unw_table *ut;
 
-	type = reg >> 5;
-	reg &= 0x1f;
-	switch (type) {
-	case 0:
-		DPF(("save location for gr[%d] at sp+%d at offset %d\n",
-		       reg, 4*spoff, t));
-		us->us_regs.rs_gr[reg].ur_save = (u_int64_t *)
-			(us->us_regs.rs_gr[12].ur_value + 4*spoff);
-		us->us_regs.rs_gr[reg].ur_when = t;
-		break;
-	case 1:
-		DPF(("save location for fr[%d] at sp+%d at offset %d\n",
-		       reg, 4*spoff, t));
-		us->us_regs.rs_fr[reg].ur_save = (struct ia64_fpreg *)
-			(us->us_regs.rs_gr[12].ur_value + 4*spoff);
-		us->us_regs.rs_fr[reg].ur_when = t;
-		break;
-	case 2:
-		DPF(("save location for br[%d] at sp+%d at offset %d\n",
-		       reg, 4*spoff, t));
-		us->us_regs.rs_br[reg].ur_save = (u_int64_t *)
-			(us->us_regs.rs_gr[12].ur_value + 4*spoff);
-		us->us_regs.rs_br[reg].ur_when = t;
-		break;
-	case 3:
-		switch (reg) {
-		case 0:
-			PROCESS_SPREL_WHEN(us, preds, spoff, t);
-			break;
-		case 1:
-			PROCESS_SPREL_WHEN(us, psp, spoff, t);
-			break;
-		case 2:
-			PROCESS_SPREL_WHEN(us, priunat, spoff, t);
-			break;
-		case 3:
-			PROCESS_SPREL_WHEN(us, br[0], spoff, t);
-			break;
-		case 4:
-			PROCESS_SPREL_WHEN(us, bsp, spoff, t);
-			break;
-		case 5:
-			PROCESS_SPREL_WHEN(us, bspstore, spoff, t);
-			break;
-		case 6:
-			PROCESS_SPREL_WHEN(us, rnat, spoff, t);
-			break;
-		case 7:
-			PROCESS_SPREL_WHEN(us, unat, spoff, t);
-			break;
-		case 8:
-			PROCESS_SPREL_WHEN(us, fpsr, spoff, t);
-			break;
-		case 9:
-			PROCESS_SPREL_WHEN(us, pfs, spoff, t);
-			break;
-		case 10:
-			PROCESS_SPREL_WHEN(us, lc, spoff, t);
-			break;
-		}
+	ut = unw_table_lookup(base);
+	if (ut != NULL) {
+		LIST_REMOVE(ut, ut_link);
+		free(ut, M_UNWIND);
+		if (bootverbose)
+			printf("UNWIND: table removed: base=%lx\n", base);
 	}
 }
 
 static void
-parse_spill_reg(struct ia64_unwind_state *us, int t,
-		int reg, int treg)
-{
-	/* not done yet  */
-}
-
-static void
-parse_spill_psprel_p(struct ia64_unwind_state *us, int t, int qp,
-		     int reg, int pspoff)
+unw_initialize(void *dummy __unused)
 {
-	/* not done yet  */
-}
-
-static void
-parse_spill_sprel_p(struct ia64_unwind_state *us, int t, int qp,
-		    int reg, int spoff)
-{
-	/* not done yet  */
-}
-
-static void
-parse_spill_reg_p(struct ia64_unwind_state *us, int t, int qp,
-		  int reg, int treg)
-{
-	/* not done yet  */
-}
-
-static void
-unwind_region(struct register_state *regs, int pc)
-{
-	int i;
-
-#define RESTORE(x)							\
-	do {								\
-		if (regs->rs_##x.ur_save				\
-		    && pc > regs->rs_##x.ur_when) {			\
-			DPF(("restoring %s\n", #x));			\
-			regs->rs_##x.ur_value = *regs->rs_##x.ur_save;	\
-		}							\
-		regs->rs_##x.ur_save = 0;				\
-		regs->rs_##x.ur_when = 0;				\
-	} while (0)
-
-#define RESTORE_INDEX(x, i)						\
-	do {								\
-		if (regs->rs_##x[i].ur_save				\
-		    && pc > regs->rs_##x[i].ur_when) {			\
-			DPF(("restoring %s[%d]\n", #x, i));		\
-			regs->rs_##x[i].ur_value =			\
-				*regs->rs_##x[i].ur_save;		\
-		}							\
-		regs->rs_##x[i].ur_save = 0;				\
-		regs->rs_##x[i].ur_when = 0;				\
-	} while (0)
-
-	if (regs->rs_stack_size) {
-		DPF(("restoring psp\n"));
-		regs->rs_psp.ur_value =
-			regs->rs_gr[12].ur_value + regs->rs_stack_size;
-		regs->rs_stack_size = 0;
-	} else {
-		RESTORE(psp);
-	}
-	RESTORE(pfs);
-	RESTORE(preds);
-	RESTORE(unat);
-	RESTORE(lc);
-	RESTORE(rnat);
-	RESTORE(bsp);
-	RESTORE(bspstore);
-	RESTORE(fpsr);
-	RESTORE(priunat);
-	for (i = 0; i < 8; i++)
-		RESTORE_INDEX(br, i);
-	for (i = 0; i < 32; i++)
-		RESTORE_INDEX(gr, i);
-	for (i = 0; i < 32; i++)
-		RESTORE_INDEX(fr, i);
-
-#undef RESTORE
-}
-
-int
-ia64_unwind_state_previous_frame(struct ia64_unwind_state *us)
-{
-	struct ia64_unwind_table *ut;
-	struct ia64_unwind_table_entry *ute;
-	struct ia64_unwind_info *ui;
-	u_int8_t *p;
-	u_int8_t *end;
-	int region = 0;	/* 0 for prologue, 1 for body */
-	int rlen = 0;
-
-	/*
-	 * Find the entry which describes this procedure.
-	 */
-	ut = find_table(us->us_ip);
-	if (!ut)
-		return ENOENT;
-	ute = find_entry(ut, us->us_ip);
-	if (!ute) {
-		/*
-		 * If there is no entry for this procedure, we assumes
-		 * its a leaf (i.e. rp and ar.pfs is enough to restore
-		 * the previous frame.
-		 */
-		goto noentry;
-	}
-
-	/*
-	 * Calculate 'pc' as the number of instructions from the start
-	 * of the procedure.
-	 */
-	us->us_pc = ((us->us_ip - (ute->ue_start + ut->ut_base)) / 16) * 3
-		+ us->us_ri;
-
-	/*
-	 * Process unwind records until we find the record which
-	 * contains the pc.
-	 */
-	ui = (struct ia64_unwind_info *) (ute->ue_info + ut->ut_base);
-	p = (u_int8_t *) ui + 8;
-	end = p + 8 * ui->ui_length;
-
-	while (us->us_pc > 0 && p < end) {
-		u_int8_t b = *p;
-
-		/*
-		 * Is this a header or a region descriptor?
-		 */
-		if ((b >> 7) == 0) {
-			/*
-			 * Header.
-			 *
-			 * Complete processing of previous region (if
-			 * any) by restoring the appropriate registers
-			 * and ajust pc to be relative to next region.
-			 */
-			unwind_region(&us->us_regs, us->us_pc);
-			us->us_pc -= rlen;
-			if (us->us_pc <= 0)
-				break;
-
-			if ((b >> 6) == 0) {
-				/* R1 */
-				region = (b >> 5) & 1;
-				rlen = b & 0x1f;
-				parse_prologue(us, rlen);
-				p++;
-			} else if ((b >> 3) == 8) {
-				/* R2 */
-				int mask, grsave;
-				mask = ((b & 7) << 1) | (p[1] >> 7);
-				grsave = p[1] & 0x7f;
-				p += 2;
-				rlen = read_uleb128(&p);
-				parse_prologue_gr(us, rlen, mask, grsave);
-			} else if ((b >> 2) == 24) {
-				/* R3 */
-				region = b & 3;
-				p += 2;
-				rlen = read_uleb128(&p);
-				parse_prologue(us, rlen);
-			} 
-		} else {
-			if (region == 0) {
-				/*
-				 * Prologue
-				 */
-				if ((b >> 5) == 4) {
-					/* P1 */
-					parse_br_mem(us, b & 0x1f);
-					p++;
-				} else if ((b >> 4) == 10) {
-					/* P2 - br_gr */
-					parse_br_gr(us,
-						    (((b & 0xf) << 1)
-						     | (p[1] >> 7)),
-						    p[1] & 0x7f);
-					p += 2;
-				} else if ((b >> 3) == 22) {
-					/* P3 */
-					int which, r;
-					which = ((b & 7) << 1) | (p[1] >> 7);
-					r = p[1] & 0x7f;
-					switch (which) {
-					case 0:
-						parse_psp_gr(us, r);
-						break;
-					case 1:
-						parse_rp_gr(us, r);
-						break;
-					case 2:
-						parse_pfs_gr(us, r);
-						break;
-					case 3:
-						parse_preds_gr(us, r);
-						break;
-					case 4:
-						parse_unat_gr(us, r);
-						break;
-					case 5:
-						parse_lc_gr(us, r);
-						break;
-					case 6:
-						parse_rp_br(us, r);
-						break;
-					case 7:
-						parse_rnat_gr(us, r);
-						break;
-					case 8:
-						parse_bsp_gr(us, r);
-						break;
-					case 9:
-						parse_bspstore_gr(us, r);
-						break;
-					case 10:
-						parse_fpsr_gr(us, r);
-						break;
-					case 11:
-						parse_priunat_gr(us, r);
-						break;
-					}
-					p += 2;
-				} else if ((b >> 0) == 184) {
-					/* P4 */
-					parse_spill_mask(us, rlen, p+1);
-					p += 1 + (rlen + 3) / 4;
-				} else if ((b >> 0) == 185) {
-					/* P5 - frgr_mem */
-					parse_frgr_mem(us,
-						       p[1] >> 4,
-						       ((p[1] & 0xf) << 16)
-						       | (p[2] << 8) | p[3]);
-					p += 3;
-				} else if ((b >> 5) == 6) {
-					/* P6 */
-					if (b & 0x10)
-						parse_gr_mem(us, b & 0xf);
-					else
-						parse_fr_mem(us, b & 0xf);
-					p++;
-				} else if ((b >> 4) == 14) {
-					/* P7 */
-					int x, y;
-					p++;
-					x = read_uleb128(&p);
-					switch (b & 0xf) {
-					case 0:
-						y = read_uleb128(&p);
-						parse_mem_stack_f(us, x, y);
-						break;
-					case 1:
-						parse_mem_stack_v(us, x);
-						break;
-					case 2:
-						parse_spill_base(us, x);
-						break;
-					case 3:
-						parse_psp_sprel(us, x);
-						break;
-					case 4:
-						parse_rp_when(us, x);
-						break;
-					case 5:
-						parse_rp_psprel(us, x);
-						break;
-					case 6:
-						parse_pfs_when(us, x);
-						break;
-					case 7:
-						parse_pfs_psprel(us, x);
-						break;
-					case 8:
-						parse_preds_when(us, x);
-						break;
-					case 9:
-						parse_preds_psprel(us, x);
-						break;
-					case 10:
-						parse_lc_when(us, x);
-						break;
-					case 11:
-						parse_lc_psprel(us, x);
-						break;
-					case 12:
-						parse_unat_when(us, x);
-						break;
-					case 13:
-						parse_unat_psprel(us, x);
-						break;
-					case 14:
-						parse_fpsr_when(us, x);
-						break;
-					case 15:
-						parse_fpsr_psprel(us, x);
-						break;
-					}
-				} else if ((b >> 0) == 240) {
-					/* P8 */
-					int which = p[1];
-					int x;
-					p += 2;
-					x = read_uleb128(&p);
-					switch (which) {
-					case 1:
-						parse_rp_sprel(us, x);
-						break;
-					case 2:
-						parse_pfs_sprel(us, x);
-						break;
-					case 3:
-						parse_preds_sprel(us, x);
-						break;
-					case 4:
-						parse_lc_sprel(us, x);
-						break;
-					case 5:
-						parse_unat_sprel(us, x);
-						break;
-					case 6:
-						parse_fpsr_sprel(us, x);
-						break;
-					case 7:
-						parse_bsp_when(us, x);
-						break;
-					case 8:
-						parse_bsp_psprel(us, x);
-						break;
-					case 9:
-						parse_bsp_sprel(us, x);
-						break;
-					case 10:
-						parse_bspstore_when(us, x);
-						break;
-					case 11:
-						parse_bspstore_psprel(us, x);
-						break;
-					case 12:
-						parse_bspstore_sprel(us, x);
-						break;
-					case 13:
-						parse_rnat_when(us, x);
-						break;
-					case 14:
-						parse_rnat_psprel(us, x);
-						break;
-					case 15:
-						parse_rnat_sprel(us, x);
-						break;
-					case 16:
-						parse_priunat_when_gr(us, x);
-						break;
-					case 17:
-						parse_priunat_psprel(us, x);
-						break;
-					case 18:
-						parse_priunat_sprel(us, x);
-						break;
-					case 19:
-						parse_priunat_when_mem(us, x);
-						break;
-					}
-				} else if ((b >> 0) == 241) {
-					/* P9 */
-					parse_gr_gr(us, p[1], p[2]);
-					p += 3;
-				} else if ((b >> 0) == 255) {
-					/* P10 (ignored) */
-					p += 3;
-				}
-			} else {
-				if ((b >> 6) == 2) {
-					/* B1 */
-					if ((b & (1 << 5)) == 0)
-						parse_label_state(us,
-								  b & 0x1f);
-					else
-						parse_copy_state(us,
-								 b & 0x1f);
-				} else if ((b >> 5) == 6) {
-					/* B2 */
-					int ecount = b & 0x1f;
-					int t;
-					p++;
-					t = read_uleb128(&p);
-					parse_epilogue(us, t, ecount);
-				} else if ((b >> 0) == 224) {
-					/* B3 */
-					int t, ecount;
-					p++;
-					t = read_uleb128(&p);
-					ecount = read_uleb128(&p);
-					parse_epilogue(us, t, ecount);
-				} else if ((b >> 4) == 15) {
-					/* B4 */
-					int label;
-					p++;
-					label = read_uleb128(&p);
-					if ((b & (1 << 3)) == 0)
-						parse_label_state(us, label);
-					else
-						parse_copy_state(us, label);
-				}
-			}
-			/*
-			 * X records can appear in both prologue and
-			 * body.
-			 */
-			if ((b >> 0) == 249) {
-				/* X1 */
-				int r, reg, t, off;
-				r = (p[1] >> 7) & 1;
-				reg = p[1] & 0x7f;
-				p += 2;
-				t = read_uleb128(&p);
-				off = read_uleb128(&p);
-				if (r == 0)
-					parse_spill_psprel(us, t, reg, off);
-				else
-					parse_spill_sprel(us, t, reg, off);
-			} else if ((b >> 0) == 250) {
-				/* X2 */
-				int reg, treg, t;
-				reg = p[1] & 0x7f;
-				treg = p[2] | ((p[1] & 0x80) << 1);
-				p += 3;
-				t = read_uleb128(&p);
-				parse_spill_reg(us, t, reg, treg);
-			} else if ((b >> 0) == 251) {
-				/* X3 */
-				int r, qp, reg, t, off;
-				r = (p[1] >> 7) & 1;
-				qp = p[1] & 0x3f;
-				reg = p[2];
-				p += 3;
-				t = read_uleb128(&p);
-				off = read_uleb128(&p);
-				if (r == 0)
-					parse_spill_psprel_p(us, t, qp,
-							     reg, off);
-				else
-					parse_spill_sprel_p(us, t, qp,
-							    reg, off);
-			} else if ((b >> 0) == 252) {
-				/* X4 */
-				int qp, reg, treg, t;
-				qp = p[1] & 0x3f;
-				reg = p[2] & 0x7f;
-				treg = p[3] | ((p[2] & 0x80) << 1);
-				p += 4;
-				t = read_uleb128(&p);
-				parse_spill_reg_p(us, t, qp, reg, treg);
-			}
-		}
-	}
-
- noentry:
-	/*
-	 * Now that we have worked out suitable values for rp, ar.pfs
-	 * and sp, we can shift state to the previous function. If we
-	 * haven't managed to figure out a new value for ip, then we
-	 * assume that the unwinding didn't work and return an error.
-	 */
-	if (us->us_ip == us->us_regs.rs_br[0].ur_value)
-		return EINVAL;
-	us->us_ip = us->us_regs.rs_br[0].ur_value;
-	us->us_ri = 0;
-	us->us_cfm = us->us_regs.rs_pfs.ur_value;
-	DPF(("new cfm is 0x%lx\n", us->us_cfm));
-	us->us_bsp = ia64_rse_previous_frame(us->us_bsp,
-					     (us->us_cfm >> 7) & 0x7f);
-	DPF(("new bsp is %p\n", us->us_bsp));
-	us->us_regs.rs_gr[12] = us->us_regs.rs_psp;
 
-	return 0;
+	LIST_INIT(&unw_tables);
+	uwx_register_alloc_cb(unw_alloc, unw_free);
 }
+SYSINIT(unwind, SI_SUB_KMEM, SI_ORDER_ANY, unw_initialize, 0);
diff --git a/sys/ia64/ia64/vm_machdep.c b/sys/ia64/ia64/vm_machdep.c
index 5c5fbff..d0ecb6e 100644
--- a/sys/ia64/ia64/vm_machdep.c
+++ b/sys/ia64/ia64/vm_machdep.c
@@ -127,170 +127,67 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku)
  * ready to run and return to user mode.
  */
 void
-cpu_fork(td1, p2, td2, flags)
-	register struct thread *td1;
-	register struct proc *p2;
-	register struct thread *td2;
-	int flags;
+cpu_fork(struct thread *td1, struct proc *p2 __unused, struct thread *td2,
+    int flags)
 {
-	struct proc *p1;
-	struct trapframe *p2tf;
-	u_int64_t bspstore, *p1bs, *p2bs, rnatloc, rnat;
+	char *stackp;
 
 	KASSERT(td1 == curthread || td1 == &thread0,
-	    ("cpu_fork: p1 not curproc and not proc0"));
+	    ("cpu_fork: td1 not curthread and not thread0"));
 
 	if ((flags & RFPROC) == 0)
 		return;
 
-	p1 = td1->td_proc;
-	td2->td_pcb = (struct pcb *)
-	    (td2->td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
-	td2->td_md.md_flags = td1->td_md.md_flags & (MDP_FPUSED | MDP_UAC_MASK);
-
-	/*
-	 * Copy floating point state from the FP chip to the PCB
-	 * if this process has state stored there.
-	 */
-	ia64_fpstate_save(td1, 0);
-
-	/*
-	 * Copy pcb and stack from proc p1 to p2.  We do this as
-	 * cheaply as possible, copying only the active part of the
-	 * stack.  The stack and pcb need to agree. Make sure that the 
-	 * new process has FEN disabled.
-	 */
-	bcopy(td1->td_pcb, td2->td_pcb, sizeof(struct pcb));
-
 	/*
-	 * Set the floating point state.
+	 * Save the preserved registers and the high FP registers in the
+	 * PCB if we're the parent (ie td1 == curthread) so that we have
+	 * a valid PCB. This also causes a RSE flush. We don't have to
+	 * do that otherwise, because there wouldn't be anything important
+	 * to save.
 	 */
-#if 0
-	if ((td2->td_pcb->pcb_fp_control & IEEE_INHERIT) == 0) {
-		td2->td_pcb->pcb_fp_control = 0;
-		td2->td_pcb->pcb_fp.fpr_cr = (FPCR_DYN_NORMAL
-						   | FPCR_INVD | FPCR_DZED
-						   | FPCR_OVFD | FPCR_INED
-						   | FPCR_UNFD);
+	if (td1 == curthread) {
+		if (savectx(td1->td_pcb) != 0)
+			panic("unexpected return from savectx()");
+		ia64_highfp_save(td1);
 	}
-#endif
 
 	/*
-	 * Arrange for a non-local goto when the new process
-	 * is started, to resume here, returning nonzero from setjmp.
+	 * create the child's kernel stack and backing store. We basicly
+	 * create an image of the parent's stack and backing store and
+	 * adjust where necessary.
 	 */
-#ifdef DIAGNOSTIC
-	if (td1 == curthread)
-		ia64_fpstate_check(td1);
-#endif
+	stackp = (char *)(td2->td_kstack + KSTACK_PAGES * PAGE_SIZE);
 
-	/*
-	 * create the child's kernel stack, from scratch.
-	 *
-	 * Pick a stack pointer, leaving room for a trapframe;
-	 * copy trapframe from parent so return to user mode
-	 * will be to right address, with correct registers. Clear the
-	 * high-fp enable for the new process so that it is forced to
-	 * load its state from the pcb.
-	 */
-	td2->td_frame = (struct trapframe *)td2->td_pcb - 1;
-	bcopy(td1->td_frame, td2->td_frame, sizeof(struct trapframe));
-	td2->td_frame->tf_cr_ipsr |= IA64_PSR_DFH;
-
-	/*
-	 * Set up return-value registers as fork() libc stub expects.
-	 */
-	p2tf = td2->td_frame;
-	if (p2tf->tf_cr_ipsr & IA64_PSR_IS) {
-		p2tf->tf_r[FRAME_R8] = 0; /* child returns zero (eax) */
-		p2tf->tf_r[FRAME_R10] = 1; /* is child (edx) */
-		td2->td_pcb->pcb_ar_eflag &= ~PSL_C; /* no error */
-	} else {
-		p2tf->tf_r[FRAME_R8] = 0; /* child's pid (linux) 	*/
-		p2tf->tf_r[FRAME_R9] = 1; /* is child (FreeBSD) 	*/
-		p2tf->tf_r[FRAME_R10] = 0; /* no error	 		*/
-	}
+	stackp -= sizeof(struct pcb);
+	td2->td_pcb = (struct pcb *)stackp;
+	bcopy(td1->td_pcb, td2->td_pcb, sizeof(struct pcb));
 
-	/*
-	 * Turn off RSE for a moment and work out our current
-	 * ar.bspstore. This assumes that td1==curthread. Also
-	 * flush dirty regs to ensure that the user's stacked
-	 * regs are written out to backing store.
-	 *
-	 * We could cope with td1!=curthread by digging values
-	 * out of its PCB but I don't see the point since
-	 * current usage only allows &thread0 when creating kernel
-	 * threads and &thread0 doesn't have any dirty regs.
-	 */
+	stackp -= sizeof(struct trapframe);
+	td2->td_frame = (struct trapframe *)stackp;
+	bcopy(td1->td_frame, td2->td_frame, sizeof(struct trapframe));
+	td2->td_frame->tf_length = sizeof(struct trapframe);
 
-	p1bs = (u_int64_t *)td1->td_kstack;
-	p2bs = (u_int64_t *)td2->td_kstack;
+	bcopy((void*)td1->td_kstack, (void*)td2->td_kstack,
+	    td2->td_frame->tf_special.ndirty);
 
-	if (td1 == curthread) {
-		__asm __volatile("mov ar.rsc=0;;");
-		__asm __volatile("flushrs;;" ::: "memory");
-		__asm __volatile("mov %0=ar.bspstore" : "=r"(bspstore));
+	/* Set-up the return values as expected by the fork() libc stub. */
+	if (td2->td_frame->tf_special.psr & IA64_PSR_IS) {
+		td2->td_frame->tf_scratch.gr8 = 0;
+		td2->td_frame->tf_scratch.gr10 = 1;
 	} else {
-		bspstore = (u_int64_t) p1bs;
+		td2->td_frame->tf_scratch.gr8 = 0;
+		td2->td_frame->tf_scratch.gr9 = 1;
+		td2->td_frame->tf_scratch.gr10 = 0;
 	}
 
-	/*
-	 * Copy enough of td1's backing store to include all
-	 * the user's stacked regs.
-	 */
-	bcopy(p1bs, p2bs, td1->td_frame->tf_ndirty);
-	/*
-	 * To calculate the ar.rnat for td2, we need to decide
-	 * if td1's ar.bspstore has advanced past the place
-	 * where the last ar.rnat which covers the user's
-	 * saved registers would be placed. If so, we read
-	 * that one from memory, otherwise we take td1's
-	 * current ar.rnat. If we are simply spawning a new kthread
-	 * from &thread0 we don't care about ar.rnat.
-	 */
-	if (td1 == curthread) {
-		rnatloc = (u_int64_t)p1bs + td1->td_frame->tf_ndirty;
-		rnatloc |= 0x1f8;
-		if (bspstore > rnatloc)
-			rnat = *(u_int64_t *) rnatloc;
-		else
-			__asm __volatile("mov %0=ar.rnat;;" : "=r"(rnat));
-
-		/*
-		 * Switch the RSE back on.
-		 */
-		__asm __volatile("mov ar.rsc=3;;");
-	} else {
-		rnat = 0;
-	}
-	
-	/*
-	 * Setup the child's pcb so that its ar.bspstore
-	 * starts just above the region which we copied. This
-	 * should work since the child will normally return
-	 * straight into exception_restore. Also initialise its
-	 * pmap to the containing proc's vmspace.
-	 */
-	td2->td_pcb->pcb_ar_bsp = (u_int64_t)p2bs + td1->td_frame->tf_ndirty;
-	td2->td_pcb->pcb_ar_rnat = rnat;
-	td2->td_pcb->pcb_ar_pfs = 0;
-	td2->td_pcb->pcb_current_pmap = (u_int64_t)
-		vmspace_pmap(td2->td_proc->p_vmspace);
+	td2->td_pcb->pcb_special.bspstore = td2->td_kstack +
+	    td2->td_frame->tf_special.ndirty;
+	td2->td_pcb->pcb_special.pfs = 0;
+	td2->td_pcb->pcb_current_pmap = vmspace_pmap(td2->td_proc->p_vmspace);
 
-	/*
-	 * Arrange for continuation at fork_return(), which
-	 * will return to exception_restore().  Note that the
-	 * child process doesn't stay in the kernel for long!
-	 *
-	 * The extra 16 bytes subtracted from sp is part of the ia64
-	 * ABI - a function can assume that the 16 bytes above sp are
-	 * available as scratch space.
-	 */
-	td2->td_pcb->pcb_sp = (u_int64_t)p2tf - 16;	
-	td2->td_pcb->pcb_r[PCB_R4] = (u_int64_t)fork_return;
-	td2->td_pcb->pcb_r[PCB_R5] = FDESC_FUNC(exception_restore);
-	td2->td_pcb->pcb_r[PCB_R6] = (u_int64_t)td2;
-	td2->td_pcb->pcb_rp = FDESC_FUNC(fork_trampoline);
+	td2->td_pcb->pcb_special.sp = (uintptr_t)stackp - 16;
+	td2->td_pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline);
+	cpu_set_fork_handler(td2, (void (*)(void*))fork_return, td2);
 }
 
 /*
@@ -305,8 +202,8 @@ cpu_set_fork_handler(td, func, arg)
 	void (*func)(void *);
 	void *arg;
 {
-	td->td_pcb->pcb_r[PCB_R4] = (u_int64_t) func;
-	td->td_pcb->pcb_r[PCB_R6] = (u_int64_t) arg;
+	td->td_frame->tf_scratch.gr2 = (u_int64_t)func;
+	td->td_frame->tf_scratch.gr3 = (u_int64_t)arg;
 }
 
 /*
@@ -315,11 +212,11 @@ cpu_set_fork_handler(td, func, arg)
  * When the proc is reaped, cpu_wait() will gc the VM state.
  */
 void
-cpu_exit(td)
-	register struct thread *td;
+cpu_exit(struct thread *td)
 {
 
-	ia64_fpstate_drop(td);
+	/* Throw away the high FP registers. */
+	ia64_highfp_drop(td);
 }
 
 void
diff --git a/sys/ia64/include/asm.h b/sys/ia64/include/asm.h
index f0a1ec2..8be7b8a 100644
--- a/sys/ia64/include/asm.h
+++ b/sys/ia64/include/asm.h
@@ -151,15 +151,22 @@ _name_	=	_value_
 label:	ASCIZ msg;				\
 	.text;
 
+
 /*
  * System call glue.
  */
-#define	SYSCALLNUM(name)			\
-	SYS_ ## name
+#define	SYSCALLNUM(name)	SYS_ ## name
+
+#define	CALLSYS_NOERROR(name)					\
+{	.mmi ;							\
+	alloc		r9 = ar.pfs, 0, 0, 8, 0 ;		\
+	mov		r31 = ar.k5 ;				\
+	mov		r10 = b0 ;; }				\
+{	.mib ;							\
+	mov		r8 = SYSCALLNUM(name) ;			\
+	mov		b7 = r31 ; 				\
+	br.call.sptk	b0 = b7 ;; }
 
-#define	CALLSYS_NOERROR(name)			\
-	mov	r15=SYSCALLNUM(name);		\
-	break	0x100000 ;;
 
 /*
  * WEAK_ALIAS: create a weak alias (ELF only).
diff --git a/sys/ia64/include/cpu.h b/sys/ia64/include/cpu.h
index 324b035..c2bd78c 100644
--- a/sys/ia64/include/cpu.h
+++ b/sys/ia64/include/cpu.h
@@ -46,28 +46,25 @@
 #ifndef _MACHINE_CPU_H_
 #define _MACHINE_CPU_H_
 
-/*
- * Exported definitions unique to Alpha cpu support.
- */
-
 #include <machine/frame.h>
 
-#define	cpu_getstack(td)	((td)->td_frame->tf_r[FRAME_SP])
-
 /*
- * Arguments to hardclock and gatherstats encapsulate the previous
- * machine state in an opaque clockframe.  One the Alpha, we use
- * what we push on an interrupt (a trapframe).
+ * Arguments to hardclock and gatherstats encapsulate the previous machine
+ * state in an opaque clockframe.
  */
 struct clockframe {
-	struct trapframe	cf_tf;
+	struct trapframe cf_tf;
 };
-#define	TRAPF_USERMODE(framep)						\
-	(((framep)->tf_cr_ipsr & IA64_PSR_CPL) == IA64_PSR_CPL_USER)
-#define	TRAPF_PC(framep)	((framep)->tf_cr_iip)
+#define	CLKF_PC(cf)		((cf)->cf_tf.tf_special.iip)
+#define	CLKF_USERMODE(cf)	((CLKF_PC(cf) >> 61) < 5)
 
-#define	CLKF_USERMODE(framep)	TRAPF_USERMODE(&(framep)->cf_tf)
-#define	CLKF_PC(framep)		TRAPF_PC(&(framep)->cf_tf)
+/* Used by signaling code. */
+#define	cpu_getstack(td)	((td)->td_frame->tf_special.sp)
+
+/* XXX */
+#define	TRAPF_PC(tf)		((tf)->tf_special.iip)
+#define	TRAPF_USERMODE(framep)						\
+	(((framep)->tf_special.psr & IA64_PSR_CPL) == IA64_PSR_CPL_USER)
 
 /*
  * CTL_MACHDEP definitions.
@@ -107,34 +104,33 @@ struct trapframe;
 extern struct rpb *hwrpb;
 extern volatile int mc_expected, mc_received;
 
-int	badaddr	(void *, size_t);
+int	badaddr(void *, size_t);
 int	badaddr_read(void *, size_t, void *);
 u_int64_t console_restart(u_int64_t, u_int64_t, u_int64_t);
-void	do_sir(void);
+int	do_ast(struct trapframe *);
 void	dumpconf(void);
-void	exception_restore(void);				/* MAGIC */
 void	frametoreg(struct trapframe *, struct reg *);
 long	fswintrberr(void);				/* MAGIC */
-int	ia64_pa_access(u_long);
+int	ia64_highfp_drop(struct thread *);
+int	ia64_highfp_load(struct thread *);
+int	ia64_highfp_save(struct thread *);
 void	ia64_init(u_int64_t, u_int64_t);
-void	ia64_fpstate_check(struct thread *p);
-void	ia64_fpstate_save(struct thread *p, int write);
-void	ia64_fpstate_drop(struct thread *p);
-void	ia64_fpstate_switch(struct thread *p);
+int	ia64_pa_access(u_long);
 void	init_prom_interface(struct rpb*);
 void	interrupt(u_int64_t, struct trapframe *);
-void	machine_check
-	(unsigned long, struct trapframe *, unsigned long, unsigned long);
+void	machine_check(unsigned long, struct trapframe *, unsigned long,
+    unsigned long);
 u_int64_t hwrpb_checksum(void);
 void	hwrpb_restart_setup(void);
 void	regdump(struct trapframe *);
 void	regtoframe(struct reg *, struct trapframe *);
 void	set_iointr(void (*)(void *, unsigned long));
 void	fork_trampoline(void);				/* MAGIC */
-void	syscall(int, u_int64_t *, struct trapframe *);
-void	trap(int vector, int imm, struct trapframe *framep);
+int	syscall(struct trapframe *);
+void	trap(int vector, struct trapframe *framep);
 void	ia64_probe_sapics(void);
 int	ia64_count_cpus(void);
+void	map_gateway_page(void);
 void	map_pal_code(void);
 void	map_port_space(void);
 void	cpu_mp_add(uint, uint, uint);
diff --git a/sys/ia64/include/db_machdep.h b/sys/ia64/include/db_machdep.h
index 9a54356..4b801f6 100644
--- a/sys/ia64/include/db_machdep.h
+++ b/sys/ia64/include/db_machdep.h
@@ -50,8 +50,8 @@ typedef struct trapframe db_regs_t;
 extern db_regs_t	ddb_regs;	/* register state */
 #define	DDB_REGS	(&ddb_regs)
 
-#define	PC_REGS(regs)	((db_addr_t)(regs)->tf_cr_iip \
-			 + (((regs)->tf_cr_ipsr >> 41) & 3))
+#define	PC_REGS(regs)	((db_addr_t)(regs)->tf_special.iip + \
+	(((regs)->tf_special.psr >> 41) & 3))
 
 #define BKPT_WRITE(addr, storage)	db_write_breakpoint(addr, storage)
 #define BKPT_CLEAR(addr, storage)	db_clear_breakpoint(addr, storage)
@@ -59,8 +59,8 @@ extern db_regs_t	ddb_regs;	/* register state */
 
 #define BKPT_SKIP			db_skip_breakpoint()
 
-#define db_clear_single_step(regs)	ddb_regs.tf_cr_ipsr &= ~IA64_PSR_SS
-#define db_set_single_step(regs)	ddb_regs.tf_cr_ipsr |= IA64_PSR_SS
+#define db_clear_single_step(regs)	ddb_regs.tf_special.psr &= ~IA64_PSR_SS
+#define db_set_single_step(regs)	ddb_regs.tf_special.psr |= IA64_PSR_SS
 
 #define	IS_BREAKPOINT_TRAP(type, code)	(type == IA64_VEC_BREAK)
 #define	IS_WATCHPOINT_TRAP(type, code)	0
diff --git a/sys/ia64/include/frame.h b/sys/ia64/include/frame.h
index a66345d..262a8bf 100644
--- a/sys/ia64/include/frame.h
+++ b/sys/ia64/include/frame.h
@@ -29,87 +29,18 @@
 #ifndef _MACHINE_FRAME_H_
 #define	_MACHINE_FRAME_H_
 
-#include <machine/reg.h>
+#include <machine/_regset.h>
 
 /*
  * Software trap, exception, and syscall frame.
  */
 struct trapframe {
-	u_int64_t		tf_flags;
+	uint64_t		tf_length;
+	uint64_t		tf_flags;
 #define FRAME_SYSCALL		1	/* syscalls use a partial trapframe */
-
-	u_int64_t		tf_cr_iip;
-	u_int64_t		tf_cr_ipsr;
-	u_int64_t		tf_cr_isr;
-	u_int64_t		tf_cr_ifa;
-	u_int64_t		tf_pr;
-	u_int64_t		tf_ar_rsc;
-	u_int64_t		tf_ar_pfs;
-	u_int64_t		tf_cr_ifs;
-	u_int64_t		tf_ar_bspstore;
-	u_int64_t		tf_ar_rnat;
-	u_int64_t		tf_ndirty;
-	u_int64_t		tf_ar_unat;
-	u_int64_t		tf_ar_ccv;
-	u_int64_t		tf_ar_fpsr;
-	u_int64_t		tf_ar_lc;
-	u_int64_t		tf_ar_ec;
-
-	u_int64_t		tf_b[8];
-
-	u_int64_t		tf_r[31];	/* don't need to save r0 */
-#define FRAME_R1		0
-#define FRAME_R2		1
-#define FRAME_R3		2
-#define FRAME_R4		3
-#define FRAME_R5		4
-#define FRAME_R6		5
-#define FRAME_R7		6
-#define FRAME_R8		7
-#define FRAME_R9		8
-#define FRAME_R10		9
-#define FRAME_R11		10
-#define FRAME_R12		11
-#define FRAME_R13		12
-#define FRAME_R14		13
-#define FRAME_R15		14
-#define FRAME_R16		15
-#define FRAME_R17		16
-#define FRAME_R18		17
-#define FRAME_R19		18
-#define FRAME_R20		19
-#define FRAME_R21		20
-#define FRAME_R22		21
-#define FRAME_R23		22
-#define FRAME_R24		23
-#define FRAME_R25		24
-#define FRAME_R26		25
-#define FRAME_R27		26
-#define FRAME_R28		27
-#define FRAME_R29		28
-#define FRAME_R30		29
-#define FRAME_R31		30
-
-#define FRAME_GP		FRAME_R1
-#define FRAME_SP		FRAME_R12
-#define FRAME_TP		FRAME_R13
-
-	/*
-	 * We rely on the compiler to save/restore f2-f5 and
-	 * f16-f31. We also tell the compiler to avoid f32-f127
-	 * completely so we don't worry about them at all.
-	 */
-	struct ia64_fpreg	tf_f[10];
-#define FRAME_F6		0
-#define FRAME_F7		1
-#define FRAME_F8		2
-#define FRAME_F9		3
-#define FRAME_F10		4
-#define FRAME_F11		5
-#define FRAME_F12		6
-#define FRAME_F13		7
-#define FRAME_F14		8
-#define FRAME_F15		9
+	struct _special		tf_special;
+	struct _caller_saved	tf_scratch;
+	struct _caller_saved_fp	tf_scratch_fp;
 };
 
 #endif /* _MACHINE_FRAME_H_ */
diff --git a/sys/ia64/include/pcb.h b/sys/ia64/include/pcb.h
index c5ee145..ac69409 100644
--- a/sys/ia64/include/pcb.h
+++ b/sys/ia64/include/pcb.h
@@ -1,4 +1,5 @@
 /*-
+ * Copyright (c) 2003 Doug Rabson
  * Copyright (c) 2000 Doug Rabson
  * All rights reserved.
  *
@@ -29,72 +30,44 @@
 #ifndef _MACHINE_PCB_H_
 #define _MACHINE_PCB_H_
 
+#include <machine/_regset.h>
+
 /*
  * PCB: process control block
  */
+struct pmap;
 struct pcb {
-	uint64_t		pcb_sp;
-	uint64_t		pcb_ar_unat;
-	uint64_t		pcb_rp;
-	uint64_t		pcb_pr;
-	struct ia64_fpreg	pcb_f[20];
-#define	PCB_F2		0
-#define	PCB_F3		1
-#define	PCB_F4		2
-#define	PCB_F5		3
-#define	PCB_F16		4
-#define	PCB_F17		5
-#define	PCB_F18		6
-#define	PCB_F19		7
-#define	PCB_F20		8
-#define	PCB_F21		9
-#define	PCB_F22		10
-#define	PCB_F23		11
-#define	PCB_F24		12
-#define	PCB_F25		13
-#define	PCB_F26		14
-#define	PCB_F27		15
-#define	PCB_F28		16
-#define	PCB_F29		17
-#define	PCB_F30		18
-#define	PCB_F31		19
-	uint64_t		pcb_r[4];
-#define	PCB_R4		0
-#define	PCB_R5		1
-#define	PCB_R6		2
-#define	PCB_R7		3
-	uint64_t		pcb_unat47;
-	uint64_t		pcb_b[5];
-#define	PCB_B1		0
-#define	PCB_B2		1
-#define	PCB_B3		2
-#define	PCB_B4		3
-#define	PCB_B5		4
-	uint64_t		pcb_ar_bsp;
-	uint64_t		pcb_ar_pfs;
-	uint64_t		pcb_ar_rnat;
-	uint64_t		pcb_ar_lc;
-
-	uint64_t		pcb_current_pmap;
-
-	uint64_t		pcb_ar_fcr;
-	uint64_t		pcb_ar_eflag;
-	uint64_t		pcb_ar_csd;
-	uint64_t		pcb_ar_ssd;
-	uint64_t		pcb_ar_fsr;
-	uint64_t		pcb_ar_fir;
-	uint64_t		pcb_ar_fdr;
-
-	/* Aligned! */
-	struct ia64_fpreg	pcb_highfp[96];	/* f32-f127 */
+	struct _special		pcb_special;
+	struct _callee_saved	pcb_preserved;
+	struct _callee_saved_fp	pcb_preserved_fp;
+	struct _high_fp		pcb_high_fp;
+	struct pcpu		*pcb_fpcpu;
+	struct pmap 		*pcb_current_pmap;
 
 	uint64_t		pcb_onfault;	/* for copy faults */
 	uint64_t		pcb_accessaddr;	/* for [fs]uswintr */
+
+#if IA32
+	uint64_t		pcb_ia32_cflg;
+	uint64_t		pcb_ia32_eflag;
+	uint64_t		pcb_ia32_fcr;
+	uint64_t		pcb_ia32_fdr;
+	uint64_t		pcb_ia32_fir;
+	uint64_t		pcb_ia32_fsr;
+#endif
 };
 
 #ifdef _KERNEL
-void restorectx(struct pcb *);
-void savectx(struct pcb *);
+
+#define	savectx(p)	swapctx(p, NULL)
+void restorectx(struct pcb *) __dead2;
+int swapctx(struct pcb *old, struct pcb *new);
+
+#if IA32
+void ia32_restorectx(struct pcb *);
+void ia32_savectx(struct pcb *);
+#endif
+
 #endif
 
 #endif /* _MACHINE_PCB_H_ */
diff --git a/sys/ia64/include/proc.h b/sys/ia64/include/proc.h
index 4daead6..01ba3e8 100644
--- a/sys/ia64/include/proc.h
+++ b/sys/ia64/include/proc.h
@@ -37,7 +37,6 @@
 
 struct mdthread {
 	u_long		md_flags;
-	void		*md_kstackvirt;	/* virtual address of td_kstack */
 	vm_offset_t	md_bspstore;	/* initial ar.bspstore */
 	register_t	md_savecrit;
 };
@@ -50,7 +49,7 @@ struct mdthread {
 #define MDP_UAC_MASK	(MDP_UAC_NOPRINT | MDP_UAC_NOFIX | MDP_UAC_SIGBUS)
 
 struct mdproc {
-	struct user	*md_uservirt;	/* virtual address of p_addr */
+	int		__dummy;	/* Avoid having an empty struct. */
 };
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/ia64/include/reg.h b/sys/ia64/include/reg.h
index 8ba6e7d..d453826 100644
--- a/sys/ia64/include/reg.h
+++ b/sys/ia64/include/reg.h
@@ -29,37 +29,18 @@
 #ifndef _MACHINE_REG_H_
 #define _MACHINE_REG_H_
 
-#ifndef _IA64_FPREG_DEFINED
-
-struct ia64_fpreg {
-	uint64_t	fpr_bits[2];
-} __aligned(16);
-
-#define _IA64_FPREG_DEFINED
-
-#endif
+#include <machine/_regset.h>
 
 struct reg {
-	uint64_t	r_gr[128];
-	uint64_t	r_br[8];
-	uint64_t	r_cfm;
-	uint64_t	r_ip;		/* Bits 0-3 encode the slot number */
-	uint64_t	r_pr;
-	uint64_t	r_psr;		/* User mask */
-	uint64_t	r_ar_rsc;
-	uint64_t	r_ar_bsp;
-	uint64_t	r_ar_bspstore;
-	uint64_t	r_ar_rnat;
-	uint64_t	r_ar_ccv;
-	uint64_t	r_ar_unat;
-	uint64_t	r_ar_fpsr;
-	uint64_t	r_ar_pfs;
-	uint64_t	r_ar_lc;
-	uint64_t	r_ar_ec;
+	struct _special		r_special;
+	struct _callee_saved	r_preserved;
+	struct _caller_saved	r_scratch;
 };
 
 struct fpreg {
-	struct ia64_fpreg fpr_regs[128];
+	struct _callee_saved_fp	fpr_preserved;
+	struct _caller_saved_fp	fpr_scratch;
+	struct _high_fp		fpr_high;
 };
 
 struct dbreg {
@@ -68,15 +49,9 @@ struct dbreg {
 };
 
 #ifdef _KERNEL
-
 struct thread;
 
-void	restorehighfp(struct ia64_fpreg *);
-void	savehighfp(struct ia64_fpreg *);
-
-/*
- * XXX these interfaces are MI, so they should be declared in a MI place.
- */
+/* XXX these interfaces are MI, so they should be declared in a MI place. */
 int	fill_regs(struct thread *, struct reg *);
 int	set_regs(struct thread *, struct reg *);
 int	fill_fpregs(struct thread *, struct fpreg *);
diff --git a/sys/ia64/include/signal.h b/sys/ia64/include/signal.h
index 1a20e01..025b979 100644
--- a/sys/ia64/include/signal.h
+++ b/sys/ia64/include/signal.h
@@ -51,62 +51,35 @@ typedef long	sig_atomic_t;
 #endif
 
 #if __XSI_VISIBLE
-/*
- * Minimum signal stack size. The current signal frame
- * for IA-64 is 2656 bytes large.
- */
+/* Minimum signal stack size. */
 #define MINSIGSTKSZ     (3072 * 4)
 #endif
 
-#if __BSD_VISIBLE
-#ifndef _IA64_FPREG_DEFINED
-
-struct ia64_fpreg {
-	unsigned long	fpr_bits[2];
-} __aligned(16);
-
-#define _IA64_FPREG_DEFINED
-
-#endif
-#endif
-
-
 /*
  * Information pushed on stack when a signal is delivered.
  * This is used by the kernel to restore state following
  * execution of the signal handler.  It is also made available
  * to the handler to allow it to restore state properly if
  * a non-standard exit is performed.
- *
- * Note that sc_regs[] and sc_fpregs[]+sc_fpcr are inline
- * representations of 'struct reg' and 'struct fpreg', respectively.
  */
 
 #if __BSD_VISIBLE
+#include <machine/_regset.h>
+
 /*
  * The sequence of the fields should match those in
  * mcontext_t. Keep them in sync!
  */
 struct sigcontext {
-	struct __sigset	sc_mask;		/* signal mask to restore */
-	unsigned long	sc_onstack;
-	unsigned long	sc_flags;
-	unsigned long	sc_nat;
-	unsigned long	sc_sp;
-	unsigned long	sc_ip;
-	unsigned long	sc_cfm;
-	unsigned long	sc_um;
-	unsigned long	sc_ar_rsc;
-	unsigned long	sc_ar_bsp;
-	unsigned long	sc_ar_rnat;
-	unsigned long	sc_ar_ccv;
-	unsigned long	sc_ar_unat;
-	unsigned long	sc_ar_fpsr;
-	unsigned long	sc_ar_pfs;
-	unsigned long	sc_pr;
-	unsigned long	sc_br[8];
-	unsigned long	sc_gr[32];
-	struct ia64_fpreg sc_fr[128];
+	struct __sigset		sc_mask;	/* signal mask to restore */
+	unsigned long		sc_onstack;
+	unsigned long		sc_flags;
+	struct _special		sc_special;
+	struct _callee_saved	sc_preserved;
+	struct _callee_saved_fp	sc_preserved_fp;
+	struct _caller_saved	sc_scratch;
+	struct _caller_saved_fp	sc_scratch_fp;
+	struct _high_fp		sc_high_fp;
 };
 #endif /* __BSD_VISIBLE */
 
diff --git a/sys/ia64/include/smp.h b/sys/ia64/include/smp.h
index e985fc0..daf5911 100644
--- a/sys/ia64/include/smp.h
+++ b/sys/ia64/include/smp.h
@@ -14,15 +14,16 @@
  */
 /* Architecture specific IPIs. */
 #define	IPI_AP_WAKEUP		0
-#define	IPI_MCA_RENDEZ		1
+#define	IPI_HIGH_FP		1
 #define	IPI_MCA_CMCV		2
-#define	IPI_TEST		3
+#define	IPI_MCA_RENDEZ		3
+#define	IPI_TEST		4
 /* Machine independent IPIs. */
-#define	IPI_AST			4
-#define	IPI_RENDEZVOUS		5
-#define	IPI_STOP		6
+#define	IPI_AST			5
+#define	IPI_RENDEZVOUS		6
+#define	IPI_STOP		7
 
-#define	IPI_COUNT		7
+#define	IPI_COUNT		8
 
 #ifndef LOCORE
 
@@ -32,6 +33,7 @@ void	ipi_all(int ipi);
 void	ipi_all_but_self(int ipi);
 void	ipi_selected(u_int64_t cpus, int ipi);
 void	ipi_self(int ipi);
+void	ipi_send(u_int64_t lid, int ipi);
 
 #endif /* !LOCORE */
 #endif /* _KERNEL */
diff --git a/sys/ia64/include/ucontext.h b/sys/ia64/include/ucontext.h
index e8660a5..fc32b76 100644
--- a/sys/ia64/include/ucontext.h
+++ b/sys/ia64/include/ucontext.h
@@ -31,35 +31,19 @@
 #ifndef _MACHINE_UCONTEXT_H_
 #define	_MACHINE_UCONTEXT_H_
 
-#define IA64_MC_FLAG_ONSTACK		0
-#define IA64_MC_FLAG_IN_SYSCALL		1
-#define IA64_MC_FLAG_FPH_VALID		2
+#include <machine/_regset.h>
 
 typedef struct __mcontext {
-	/*
-	 * These fields must match the definition
-	 * of struct sigcontext. That way we can support
-	 * struct sigcontext and ucontext_t at the same
-	 * time.
-	 */
-	long	mc_onstack;		/* XXX - sigcontext compat. */
-	unsigned long	mc_flags;
-	unsigned long	mc_nat;
-	unsigned long	mc_sp;
-	unsigned long	mc_ip;
-	unsigned long	mc_cfm;
-	unsigned long	mc_um;
-	unsigned long	mc_ar_rsc;
-	unsigned long	mc_ar_bsp;
-	unsigned long	mc_ar_rnat;
-	unsigned long	mc_ar_ccv;
-	unsigned long	mc_ar_unat;
-	unsigned long	mc_ar_fpsr;
-	unsigned long	mc_ar_pfs;
-	unsigned long	mc_pr;
-	unsigned long	mc_br[8];
-	unsigned long	mc_gr[32];
-	struct ia64_fpreg mc_fr[128];
+	uint64_t		mc_flags;
+#define	IA64_MC_FLAGS_SCRATCH_VALID	1
+#define	IA64_MC_FLAGS_HIGHFP_VALID	2
+	uint64_t		_reserved_;
+	struct _special		mc_special;
+	struct _callee_saved	mc_preserved;
+	struct _callee_saved_fp	mc_preserved_fp;
+	struct _caller_saved	mc_scratch;
+	struct _caller_saved_fp	mc_scratch_fp;
+	struct _high_fp		mc_high_fp;
 } mcontext_t;
 
 #endif /* !_MACHINE_UCONTEXT_H_ */
diff --git a/sys/ia64/include/unwind.h b/sys/ia64/include/unwind.h
index 3509c05..1cf9f4a 100644
--- a/sys/ia64/include/unwind.h
+++ b/sys/ia64/include/unwind.h
@@ -26,13 +26,25 @@
  * $FreeBSD$
  */
 
-int ia64_add_unwind_table(vm_offset_t, vm_offset_t, vm_offset_t);
-void ia64_delete_unwind_table(vm_offset_t);
+#ifndef _MACHINE_UNWIND_H_
+#define	_MACHINE_UNWIND_H_
 
-struct ia64_unwind_state *ia64_create_unwind_state(struct trapframe *framep);
-void ia64_free_unwind_state(struct ia64_unwind_state *us);
-u_int64_t ia64_unwind_state_get_ip(struct ia64_unwind_state *us);
-u_int64_t ia64_unwind_state_get_sp(struct ia64_unwind_state *us);
-u_int64_t ia64_unwind_state_get_cfm(struct ia64_unwind_state *us);
-u_int64_t *ia64_unwind_state_get_bsp(struct ia64_unwind_state *us);
-int ia64_unwind_state_previous_frame(struct ia64_unwind_state *us);
+struct uwx_env;
+
+struct unw_regstate {
+	struct trapframe *frame;
+	struct uwx_env	*env;
+	uint64_t	keyval[8];
+};
+
+int unw_create(struct unw_regstate *s, struct trapframe *tf);
+int unw_step(struct unw_regstate *s);
+
+int unw_get_bsp(struct unw_regstate *s, uint64_t *r);
+int unw_get_cfm(struct unw_regstate *s, uint64_t *r);
+int unw_get_ip(struct unw_regstate *s, uint64_t *r);
+
+int unw_table_add(uint64_t, uint64_t, uint64_t);
+void unw_table_remove(uint64_t);
+
+#endif /* _MACHINE_UNWIND_H_ */
diff --git a/sys/ia64/include/vmparam.h b/sys/ia64/include/vmparam.h
index b9004db..69480a3 100644
--- a/sys/ia64/include/vmparam.h
+++ b/sys/ia64/include/vmparam.h
@@ -50,11 +50,10 @@
 /*
  * USRTEXT is the start of the user text/data space, while USRSTACK
  * is the top (end) of the user stack.  Immediately above the user stack
- * resides the user structure, which is UPAGES long and contains the
- * kernel stack.
+ * resides the syscall gateway page.
  */
 #define	USRTEXT		CLBYTES
-#define	USRSTACK	VM_MAXUSER_ADDRESS
+#define	USRSTACK	VM_MAX_ADDRESS
 
 /*
  * Virtual memory related constants, all in bytes
@@ -140,12 +139,13 @@
 
 /* user/kernel map constants */
 #define VM_MIN_ADDRESS		0
-#define VM_MAXUSER_ADDRESS	IA64_RR_BASE(5)
-#define VM_MAX_ADDRESS		VM_MAXUSER_ADDRESS
-#define VM_MIN_KERNEL_ADDRESS	IA64_RR_BASE(5)
+#define	VM_MAX_ADDRESS		IA64_RR_BASE(5)
+#define	VM_GATEWAY_SIZE		PAGE_SIZE
+#define	VM_MAXUSER_ADDRESS	(VM_MAX_ADDRESS + VM_GATEWAY_SIZE)
+#define	VM_MIN_KERNEL_ADDRESS	VM_MAXUSER_ADDRESS
 #define VM_MAX_KERNEL_ADDRESS	(IA64_RR_BASE(6) - 1)
 
-#define	KERNBASE		(VM_MIN_KERNEL_ADDRESS)
+#define	KERNBASE		VM_MAX_ADDRESS
 
 /* virtual sizes (bytes) for various kernel submaps */
 #ifndef VM_KMEM_SIZE