Revamp of the syscall path, exception and context handling. The

prime objectives are:
o  Implement a syscall path based on the epc inststruction (see
   sys/ia64/ia64/syscall.s).
o  Revisit the places were we need to save and restore registers
   and define those contexts in terms of the register sets (see
   sys/ia64/include/_regset.h).

Secundairy objectives:
o  Remove the requirement to use contigmalloc for kernel stacks.
o  Better handling of the high FP registers for SMP systems.
o  Switch to the new cpu_switch() and cpu_throw() semantics.
o  Add a good unwinder to reconstruct contexts for the rare
   cases we need to (see sys/contrib/ia64/libuwx)

Many files are affected by this change. Functionally it boils
down to:
o  The EPC syscall doesn't preserve registers it does not need
   to preserve and places the arguments differently on the stack.
   This affects libc and truss.
o  The address of the kernel page directory (kptdir) had to
   be unstaticized for use by the nested TLB fault handler.
   The name has been changed to ia64_kptdir to avoid conflicts.
   The renaming affects libkvm.
o  The trapframe only contains the special registers and the
   scratch registers. For syscalls using the EPC syscall path
   no scratch registers are saved. This affects all places where
   the trapframe is accessed. Most notably the unaligned access
   handler, the signal delivery code and the debugger.
o  Context switching only partly saves the special registers
   and the preserved registers. This affects cpu_switch() and
   triggered the move to the new semantics, which additionally
   affects cpu_throw().
o  The high FP registers are either in the PCB or on some
   CPU. context switching for them is done lazily. This affects
   trap().
o  The mcontext has room for all registers, but not all of them
   have to be defined in all cases. This mostly affects signal
   delivery code now. The *context syscalls are as of yet still
   unimplemented.

Many details went into the removal of the requirement to use
contigmalloc for kernel stacks. The details are mostly CPU
specific and limited to exception_save() and exception_restore().
The few places where we create, destroy or switch stacks were
mostly simplified by not having to construct physical addresses
and additionally saving the virtual addresses for later use.

Besides more efficient context saving and restoring, which of
course yields a noticable speedup, this also fixes the dreaded
SMP bootup problem as a side-effect. The details of which are
still not fully understood.

This change includes all the necessary backward compatibility
code to have it handle older userland binaries that use the
break instruction for syscalls. Support for break-based syscalls
has been pessimized in favor of a clean implementation. Due to
the overall better performance of the kernel, this will still
be notived as an improvement if it's noticed at all.

Approved by: re@ (jhb)

3 files changed, 148 insertions, 113 deletions

diff --git a/sys/ia64/ia32/ia32_signal.c b/sys/ia64/ia32/ia32_signal.c
index 18f4ee2..d9066d4 100644
--- a/sys/ia64/ia32/ia32_signal.c
+++ b/sys/ia64/ia32/ia32_signal.c
@@ -52,6 +52,10 @@
 #include <sys/vnode.h>
 #include <sys/imgact_elf.h>
 
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+
 #include <vm/vm.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_param.h>
@@ -64,8 +68,6 @@
 #include <i386/include/psl.h>
 #include <i386/include/segments.h>
 #include <i386/include/specialreg.h>
-#include <machine/frame.h>
-#include <machine/md_var.h>
 
 static register_t *ia32_copyout_strings(struct image_params *imgp);
 static void ia32_setregs(struct thread *td, u_long entry, u_long stack,
@@ -84,9 +86,9 @@ static char ia32_sigcode[] = {
 	0x50,				/* pushl %eax */
 	0xcd, 0x80,			/* int $0x80 */
 	0xeb, 0xfe,			/* 0: jmp 0b */
-	0, 0, 0, 0
+	0
 };
-static int ia32_szsigcode = sizeof(ia32_sigcode) & ~3;
+static int ia32_szsigcode = sizeof(ia32_sigcode);
 
 struct sysentvec ia32_freebsd_sysvec = {
 	SYS_MAXSYSCALL,
@@ -105,8 +107,8 @@ struct sysentvec ia32_freebsd_sysvec = {
 	"FreeBSD ELF",
 	elf32_coredump,
 	NULL,
-	MINSIGSTKSZ,
-	4096,
+	IA32_MINSIGSTKSZ,
+	IA32_PAGE_SIZE,
 	0,
 	IA32_USRSTACK,
 	IA32_USRSTACK,
@@ -145,8 +147,8 @@ ia32_copyout_strings(struct image_params *imgp)
 	 */
 	arginfo = (struct ia32_ps_strings *)IA32_PS_STRINGS;
 	szsigcode = *(imgp->proc->p_sysent->sv_szsigcode);
-	destp =	(caddr_t)arginfo - szsigcode - SPARE_USRSPACE -
-		roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
+	destp =	(caddr_t)arginfo - szsigcode - IA32_USRSPACE -
+	    roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
 
 	/*
 	 * install sigcode
@@ -185,6 +187,7 @@ ia32_copyout_strings(struct image_params *imgp)
 	/*
 	 * vectp also becomes our initial stack base
 	 */
+	vectp = (void*)((uintptr_t)vectp & ~15);
 	stack_base = vectp;
 
 	stringp = imgp->stringbase;
@@ -237,60 +240,45 @@ ia32_copyout_strings(struct image_params *imgp)
 static void
 ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 {
-	struct trapframe *frame = td->td_frame;
+	struct trapframe *tf = td->td_frame;
 	vm_offset_t gdt, ldt;
 	u_int64_t codesel, datasel, ldtsel;
 	u_int64_t codeseg, dataseg, gdtseg, ldtseg;
 	struct segment_descriptor desc;
 	struct vmspace *vmspace = td->td_proc->p_vmspace;
 
-	/*
-	 * Make sure that we restore the entire trapframe after an
-	 * execve.
-	 */
-	frame->tf_flags &= ~FRAME_SYSCALL;
-
-	bzero(frame->tf_r, sizeof(frame->tf_r));
-	bzero(frame->tf_f, sizeof(frame->tf_f));
-
-	frame->tf_cr_iip = entry;
-	frame->tf_cr_ipsr = (IA64_PSR_IC
-			     | IA64_PSR_I
-			     | IA64_PSR_IT
-			     | IA64_PSR_DT
-			     | IA64_PSR_RT
-			     | IA64_PSR_DFH
-			     | IA64_PSR_IS
-			     | IA64_PSR_BN
-			     | IA64_PSR_CPL_USER);
-	frame->tf_r[FRAME_R12] = stack;
+	exec_setregs(td, entry, stack, ps_strings);
+
+	/* Non-syscall frames are cleared by exec_setregs() */
+	if (tf->tf_flags & FRAME_SYSCALL) {
+		bzero(&tf->tf_scratch, sizeof(tf->tf_scratch));
+		bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp));
+	} else
+		tf->tf_special.ndirty = 0;
+
+	tf->tf_special.psr |= IA64_PSR_IS;
+	tf->tf_special.sp = stack;
+
+	/* Point the RSE backstore to something harmless. */
+	tf->tf_special.bspstore = (IA32_PS_STRINGS - ia32_szsigcode -
+	    IA32_USRSPACE + 15) & ~15;
 
 	codesel = LSEL(LUCODE_SEL, SEL_UPL);
 	datasel = LSEL(LUDATA_SEL, SEL_UPL);
 	ldtsel = GSEL(GLDT_SEL, SEL_UPL);
 
-#if 1
-	frame->tf_r[FRAME_R16] = (datasel << 48) | (datasel << 32)
-		| (datasel << 16) | datasel;
-	frame->tf_r[FRAME_R17] = (ldtsel << 32) | (datasel << 16) | codesel;
-#else
-	frame->tf_r[FRAME_R16] = datasel;
-	frame->tf_r[FRAME_R17] = codesel;
-	frame->tf_r[FRAME_R18] = datasel;
-	frame->tf_r[FRAME_R19] = datasel;
-	frame->tf_r[FRAME_R20] = datasel;
-	frame->tf_r[FRAME_R21] = datasel;
-	frame->tf_r[FRAME_R22] = ldtsel;
-#endif
+	/* Setup ia32 segment registers. */
+	tf->tf_scratch.gr16 = (datasel << 48) | (datasel << 32) |
+	    (datasel << 16) | datasel;
+	tf->tf_scratch.gr17 = (ldtsel << 32) | (datasel << 16) | codesel;
 
 	/*
 	 * Build the GDT and LDT.
 	 */
 	gdt = IA32_USRSTACK;
-	vm_map_find(&vmspace->vm_map, 0, 0,
-		    &gdt, PAGE_SIZE, 0,
-		    VM_PROT_ALL, VM_PROT_ALL, 0);
-	ldt = gdt + 4096;
+	vm_map_find(&vmspace->vm_map, 0, 0, &gdt, IA32_PAGE_SIZE << 1, 0,
+	    VM_PROT_ALL, VM_PROT_ALL, 0);
+	ldt = gdt + IA32_PAGE_SIZE;
 
 	desc.sd_lolimit = 8*NLDT-1;
 	desc.sd_lobase = ldt & 0xffffff;
@@ -330,12 +318,13 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (1L << 62) /* 32 bits */
 		+ (1L << 63); /* page granularity */
-	ia64_set_csd(codeseg);
-	ia64_set_ssd(dataseg);
-	frame->tf_r[FRAME_R24] = dataseg; /* ESD */
-	frame->tf_r[FRAME_R27] = dataseg; /* DSD */
-	frame->tf_r[FRAME_R28] = dataseg; /* FSD */
-	frame->tf_r[FRAME_R29] = dataseg; /* GSD */
+
+	tf->tf_scratch.csd = codeseg;
+	tf->tf_scratch.ssd = dataseg;
+	tf->tf_scratch.gr24 = dataseg; /* ESD */
+	tf->tf_scratch.gr27 = dataseg; /* DSD */
+	tf->tf_scratch.gr28 = dataseg; /* FSD */
+	tf->tf_scratch.gr29 = dataseg; /* GSD */
 
 	gdtseg = gdt		/* base */
 		+ ((8L*NGDT - 1) << 32) /* limit */
@@ -351,13 +340,16 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (0L << 62) /* 16 bits */
 		+ (0L << 63); /* byte granularity */
-	frame->tf_r[FRAME_R30] = ldtseg; /* LDTD */
-	frame->tf_r[FRAME_R31] = gdtseg; /* GDTD */
 
+	tf->tf_scratch.gr30 = ldtseg; /* LDTD */
+	tf->tf_scratch.gr31 = gdtseg; /* GDTD */
+
+	/* Set ia32 control registers on this processor. */
+	ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM | CR4_FXSR) << 32));
 	ia64_set_eflag(PSL_USER);
 
 	/* PS_STRINGS value for BSD/OS binaries.  It is 0 for non-BSD/OS. */
-	frame->tf_r[FRAME_R11] = IA32_PS_STRINGS;
+	tf->tf_scratch.gr11 = IA32_PS_STRINGS;
 
 	/*
 	 * XXX - Linux emulator
@@ -366,3 +358,27 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 	 */
 	td->td_retval[1] = 0;
 }
+
+void
+ia32_restorectx(struct pcb *pcb)
+{
+
+	ia64_set_cflg(pcb->pcb_ia32_cflg);
+	ia64_set_eflag(pcb->pcb_ia32_eflag);
+	ia64_set_fcr(pcb->pcb_ia32_fcr);
+	ia64_set_fdr(pcb->pcb_ia32_fdr);
+	ia64_set_fir(pcb->pcb_ia32_fir);
+	ia64_set_fsr(pcb->pcb_ia32_fsr);
+}
+
+void
+ia32_savectx(struct pcb *pcb)
+{
+
+	pcb->pcb_ia32_cflg = ia64_get_cflg();
+	pcb->pcb_ia32_eflag = ia64_get_eflag();
+	pcb->pcb_ia32_fcr = ia64_get_fcr();
+	pcb->pcb_ia32_fdr = ia64_get_fdr();
+	pcb->pcb_ia32_fir = ia64_get_fir();
+	pcb->pcb_ia32_fsr = ia64_get_fsr();
+}
diff --git a/sys/ia64/ia32/ia32_sysvec.c b/sys/ia64/ia32/ia32_sysvec.c
index 18f4ee2..d9066d4 100644
--- a/sys/ia64/ia32/ia32_sysvec.c
+++ b/sys/ia64/ia32/ia32_sysvec.c
@@ -52,6 +52,10 @@
 #include <sys/vnode.h>
 #include <sys/imgact_elf.h>
 
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+
 #include <vm/vm.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_param.h>
@@ -64,8 +68,6 @@
 #include <i386/include/psl.h>
 #include <i386/include/segments.h>
 #include <i386/include/specialreg.h>
-#include <machine/frame.h>
-#include <machine/md_var.h>
 
 static register_t *ia32_copyout_strings(struct image_params *imgp);
 static void ia32_setregs(struct thread *td, u_long entry, u_long stack,
@@ -84,9 +86,9 @@ static char ia32_sigcode[] = {
 	0x50,				/* pushl %eax */
 	0xcd, 0x80,			/* int $0x80 */
 	0xeb, 0xfe,			/* 0: jmp 0b */
-	0, 0, 0, 0
+	0
 };
-static int ia32_szsigcode = sizeof(ia32_sigcode) & ~3;
+static int ia32_szsigcode = sizeof(ia32_sigcode);
 
 struct sysentvec ia32_freebsd_sysvec = {
 	SYS_MAXSYSCALL,
@@ -105,8 +107,8 @@ struct sysentvec ia32_freebsd_sysvec = {
 	"FreeBSD ELF",
 	elf32_coredump,
 	NULL,
-	MINSIGSTKSZ,
-	4096,
+	IA32_MINSIGSTKSZ,
+	IA32_PAGE_SIZE,
 	0,
 	IA32_USRSTACK,
 	IA32_USRSTACK,
@@ -145,8 +147,8 @@ ia32_copyout_strings(struct image_params *imgp)
 	 */
 	arginfo = (struct ia32_ps_strings *)IA32_PS_STRINGS;
 	szsigcode = *(imgp->proc->p_sysent->sv_szsigcode);
-	destp =	(caddr_t)arginfo - szsigcode - SPARE_USRSPACE -
-		roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
+	destp =	(caddr_t)arginfo - szsigcode - IA32_USRSPACE -
+	    roundup((ARG_MAX - imgp->stringspace), sizeof(char *));
 
 	/*
 	 * install sigcode
@@ -185,6 +187,7 @@ ia32_copyout_strings(struct image_params *imgp)
 	/*
 	 * vectp also becomes our initial stack base
 	 */
+	vectp = (void*)((uintptr_t)vectp & ~15);
 	stack_base = vectp;
 
 	stringp = imgp->stringbase;
@@ -237,60 +240,45 @@ ia32_copyout_strings(struct image_params *imgp)
 static void
 ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 {
-	struct trapframe *frame = td->td_frame;
+	struct trapframe *tf = td->td_frame;
 	vm_offset_t gdt, ldt;
 	u_int64_t codesel, datasel, ldtsel;
 	u_int64_t codeseg, dataseg, gdtseg, ldtseg;
 	struct segment_descriptor desc;
 	struct vmspace *vmspace = td->td_proc->p_vmspace;
 
-	/*
-	 * Make sure that we restore the entire trapframe after an
-	 * execve.
-	 */
-	frame->tf_flags &= ~FRAME_SYSCALL;
-
-	bzero(frame->tf_r, sizeof(frame->tf_r));
-	bzero(frame->tf_f, sizeof(frame->tf_f));
-
-	frame->tf_cr_iip = entry;
-	frame->tf_cr_ipsr = (IA64_PSR_IC
-			     | IA64_PSR_I
-			     | IA64_PSR_IT
-			     | IA64_PSR_DT
-			     | IA64_PSR_RT
-			     | IA64_PSR_DFH
-			     | IA64_PSR_IS
-			     | IA64_PSR_BN
-			     | IA64_PSR_CPL_USER);
-	frame->tf_r[FRAME_R12] = stack;
+	exec_setregs(td, entry, stack, ps_strings);
+
+	/* Non-syscall frames are cleared by exec_setregs() */
+	if (tf->tf_flags & FRAME_SYSCALL) {
+		bzero(&tf->tf_scratch, sizeof(tf->tf_scratch));
+		bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp));
+	} else
+		tf->tf_special.ndirty = 0;
+
+	tf->tf_special.psr |= IA64_PSR_IS;
+	tf->tf_special.sp = stack;
+
+	/* Point the RSE backstore to something harmless. */
+	tf->tf_special.bspstore = (IA32_PS_STRINGS - ia32_szsigcode -
+	    IA32_USRSPACE + 15) & ~15;
 
 	codesel = LSEL(LUCODE_SEL, SEL_UPL);
 	datasel = LSEL(LUDATA_SEL, SEL_UPL);
 	ldtsel = GSEL(GLDT_SEL, SEL_UPL);
 
-#if 1
-	frame->tf_r[FRAME_R16] = (datasel << 48) | (datasel << 32)
-		| (datasel << 16) | datasel;
-	frame->tf_r[FRAME_R17] = (ldtsel << 32) | (datasel << 16) | codesel;
-#else
-	frame->tf_r[FRAME_R16] = datasel;
-	frame->tf_r[FRAME_R17] = codesel;
-	frame->tf_r[FRAME_R18] = datasel;
-	frame->tf_r[FRAME_R19] = datasel;
-	frame->tf_r[FRAME_R20] = datasel;
-	frame->tf_r[FRAME_R21] = datasel;
-	frame->tf_r[FRAME_R22] = ldtsel;
-#endif
+	/* Setup ia32 segment registers. */
+	tf->tf_scratch.gr16 = (datasel << 48) | (datasel << 32) |
+	    (datasel << 16) | datasel;
+	tf->tf_scratch.gr17 = (ldtsel << 32) | (datasel << 16) | codesel;
 
 	/*
 	 * Build the GDT and LDT.
 	 */
 	gdt = IA32_USRSTACK;
-	vm_map_find(&vmspace->vm_map, 0, 0,
-		    &gdt, PAGE_SIZE, 0,
-		    VM_PROT_ALL, VM_PROT_ALL, 0);
-	ldt = gdt + 4096;
+	vm_map_find(&vmspace->vm_map, 0, 0, &gdt, IA32_PAGE_SIZE << 1, 0,
+	    VM_PROT_ALL, VM_PROT_ALL, 0);
+	ldt = gdt + IA32_PAGE_SIZE;
 
 	desc.sd_lolimit = 8*NLDT-1;
 	desc.sd_lobase = ldt & 0xffffff;
@@ -330,12 +318,13 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (1L << 62) /* 32 bits */
 		+ (1L << 63); /* page granularity */
-	ia64_set_csd(codeseg);
-	ia64_set_ssd(dataseg);
-	frame->tf_r[FRAME_R24] = dataseg; /* ESD */
-	frame->tf_r[FRAME_R27] = dataseg; /* DSD */
-	frame->tf_r[FRAME_R28] = dataseg; /* FSD */
-	frame->tf_r[FRAME_R29] = dataseg; /* GSD */
+
+	tf->tf_scratch.csd = codeseg;
+	tf->tf_scratch.ssd = dataseg;
+	tf->tf_scratch.gr24 = dataseg; /* ESD */
+	tf->tf_scratch.gr27 = dataseg; /* DSD */
+	tf->tf_scratch.gr28 = dataseg; /* FSD */
+	tf->tf_scratch.gr29 = dataseg; /* GSD */
 
 	gdtseg = gdt		/* base */
 		+ ((8L*NGDT - 1) << 32) /* limit */
@@ -351,13 +340,16 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 		+ (1L << 59) /* present */
 		+ (0L << 62) /* 16 bits */
 		+ (0L << 63); /* byte granularity */
-	frame->tf_r[FRAME_R30] = ldtseg; /* LDTD */
-	frame->tf_r[FRAME_R31] = gdtseg; /* GDTD */
 
+	tf->tf_scratch.gr30 = ldtseg; /* LDTD */
+	tf->tf_scratch.gr31 = gdtseg; /* GDTD */
+
+	/* Set ia32 control registers on this processor. */
+	ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM | CR4_FXSR) << 32));
 	ia64_set_eflag(PSL_USER);
 
 	/* PS_STRINGS value for BSD/OS binaries.  It is 0 for non-BSD/OS. */
-	frame->tf_r[FRAME_R11] = IA32_PS_STRINGS;
+	tf->tf_scratch.gr11 = IA32_PS_STRINGS;
 
 	/*
 	 * XXX - Linux emulator
@@ -366,3 +358,27 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings)
 	 */
 	td->td_retval[1] = 0;
 }
+
+void
+ia32_restorectx(struct pcb *pcb)
+{
+
+	ia64_set_cflg(pcb->pcb_ia32_cflg);
+	ia64_set_eflag(pcb->pcb_ia32_eflag);
+	ia64_set_fcr(pcb->pcb_ia32_fcr);
+	ia64_set_fdr(pcb->pcb_ia32_fdr);
+	ia64_set_fir(pcb->pcb_ia32_fir);
+	ia64_set_fsr(pcb->pcb_ia32_fsr);
+}
+
+void
+ia32_savectx(struct pcb *pcb)
+{
+
+	pcb->pcb_ia32_cflg = ia64_get_cflg();
+	pcb->pcb_ia32_eflag = ia64_get_eflag();
+	pcb->pcb_ia32_fcr = ia64_get_fcr();
+	pcb->pcb_ia32_fdr = ia64_get_fdr();
+	pcb->pcb_ia32_fir = ia64_get_fir();
+	pcb->pcb_ia32_fsr = ia64_get_fsr();
+}
diff --git a/sys/ia64/ia32/ia32_util.h b/sys/ia64/ia32/ia32_util.h
index 23f2aba..a0b21b0 100644
--- a/sys/ia64/ia32/ia32_util.h
+++ b/sys/ia64/ia32/ia32_util.h
@@ -44,8 +44,11 @@ struct ia32_ps_strings {
 	int	ps_nenvstr;	/* the number of environment strings */
 };
 
-#define IA32_USRSTACK	(4L*1024*1024*1024 - PAGE_SIZE)
-#define IA32_PS_STRINGS	(IA32_USRSTACK - sizeof(struct ia32_ps_strings))
+#define	IA32_MINSIGSTKSZ	2048
+#define	IA32_PAGE_SIZE		4096
+#define	IA32_USRSTACK	(2L*1024*1024*1024 - IA32_PAGE_SIZE*2)
+#define	IA32_PS_STRINGS	(IA32_USRSTACK - sizeof(struct ia32_ps_strings))
+#define	IA32_USRSPACE		IA32_PAGE_SIZE
 
 static __inline caddr_t stackgap_init(void);
 static __inline void *stackgap_alloc(caddr_t *, size_t);
@@ -54,7 +57,7 @@ static __inline caddr_t
 stackgap_init()
 {
 #define	szsigcode (*(curproc->p_sysent->sv_szsigcode))
-	return (caddr_t)(((caddr_t)IA32_PS_STRINGS) - szsigcode - SPARE_USRSPACE);
+	return (((caddr_t)IA32_PS_STRINGS) - szsigcode - IA32_USRSPACE);
 #undef szsigcode
 }