diff options
46 files changed, 3297 insertions, 5813 deletions
diff --git a/lib/libc/ia64/sys/brk.S b/lib/libc/ia64/sys/brk.S index d325886..fee1728 100644 --- a/lib/libc/ia64/sys/brk.S +++ b/lib/libc/ia64/sys/brk.S @@ -44,12 +44,14 @@ ENTRY(brk, 1) add r14=@ltoff(minbrk),gp ;; ld8 r14=[r14] ;; ld8 r14=[r14] ;; - cmp.ltu p6,p0=in0,r14 ;; -(p6) mov in0=r14 + cmp.ltu p6,p0=r32,r14 ;; +(p6) mov r32=r14 ;; + st8 [sp]=r32 CALLSYS_ERROR(break) + ld8 r15=[sp] add r14=@ltoff(curbrk),gp ;; ld8 r14=[r14] ;; - st8 [r14]=in0 + st8 [r14]=r15 mov ret0=0 br.ret.sptk.few rp END(brk) diff --git a/lib/libc/ia64/sys/pipe.S b/lib/libc/ia64/sys/pipe.S index 0b2600c..a6413df 100644 --- a/lib/libc/ia64/sys/pipe.S +++ b/lib/libc/ia64/sys/pipe.S @@ -32,10 +32,16 @@ __FBSDID("$FreeBSD$"); #include "SYS.h" -SYSCALL(pipe) - .regstk 1,0,0,0 - st4 [in0]=ret0,4 ;; - st4 [in0]=ret1 - mov ret0=0 +ENTRY(__sys_pipe, 1) + WEAK_ALIAS(pipe, __sys_pipe) + WEAK_ALIAS(_pipe, __sys_pipe) + st8 [sp]=r32 + CALLSYS_ERROR(pipe) + ld8 r14=[sp] + ;; + st4 [r14]=ret0,4 + ;; + st4 [r14]=ret1 + mov ret0=0 br.ret.sptk.few rp -END(pipe) +END(__sys_pipe) diff --git a/lib/libc/ia64/sys/sbrk.S b/lib/libc/ia64/sys/sbrk.S index 1dc4cf2..98b5ce9 100644 --- a/lib/libc/ia64/sys/sbrk.S +++ b/lib/libc/ia64/sys/sbrk.S @@ -43,18 +43,21 @@ ENTRY(sbrk, 1) add r14 = @ltoff(curbrk), gp ;; ld8 r14 = [r14] - cmp.eq p6, p0 = in0, r0 + cmp.eq p6, p0 = r32, r0 ;; ld8 ret0 = [r14] (p6) br.ret.sptk.few rp ;; - add in0 = ret0, in0 + add r32 = ret0, r32 + ;; + st8 [sp] = r32 CALLSYS_ERROR(break) + ld8 r15 = [sp] add r14 = @ltoff(curbrk), gp ;; ld8 r14 = [r14] ;; ld8 ret0 = [r14] - st8 [r14] = in0 + st8 [r14] = r15 br.ret.sptk.few rp END(sbrk) diff --git a/lib/libkvm/kvm_ia64.c b/lib/libkvm/kvm_ia64.c index b0bdeee..9649190 100644 --- a/lib/libkvm/kvm_ia64.c +++ b/lib/libkvm/kvm_ia64.c @@ -148,7 +148,7 @@ _kvm_initvtop(kvm_t *kd) * addresses/values. */ - nlist[0].n_name = "kptdir"; + nlist[0].n_name = "ia64_kptdir"; nlist[1].n_name = 0; if (kvm_nlist(kd, nlist) != 0) { diff --git a/sys/conf/Makefile.ia64 b/sys/conf/Makefile.ia64 index 04bf91f..6015c95 100644 --- a/sys/conf/Makefile.ia64 +++ b/sys/conf/Makefile.ia64 @@ -32,6 +32,8 @@ S= ../../.. .endif .include "$S/conf/kern.pre.mk" +INCLUDES+= -I$S/contrib/ia64/libuwx/src + CFLAGS+= -mconstant-gp ASM_CFLAGS= -x assembler-with-cpp -Wa,-x -DLOCORE ${CFLAGS} diff --git a/sys/conf/files.ia64 b/sys/conf/files.ia64 index 7fccde8..24e23e7 100644 --- a/sys/conf/files.ia64 +++ b/sys/conf/files.ia64 @@ -18,6 +18,19 @@ atkbdmap.h optional atkbd_dflt_keymap \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" # +contrib/ia64/libuwx/src/uwx_bstream.c standard +contrib/ia64/libuwx/src/uwx_context.c standard +contrib/ia64/libuwx/src/uwx_env.c standard +contrib/ia64/libuwx/src/uwx_scoreboard.c standard +#contrib/ia64/libuwx/src/uwx_self.c standard +#contrib/ia64/libuwx/src/uwx_self_context.s standard +contrib/ia64/libuwx/src/uwx_step.c standard +contrib/ia64/libuwx/src/uwx_str.c standard +contrib/ia64/libuwx/src/uwx_swap.c standard +contrib/ia64/libuwx/src/uwx_trace.c standard +#contrib/ia64/libuwx/src/uwx_ttrace.c standard +contrib/ia64/libuwx/src/uwx_uinfo.c standard +contrib/ia64/libuwx/src/uwx_utable.c standard crypto/blowfish/bf_enc.c optional crypto crypto/blowfish/bf_enc.c optional ipsec ipsec_esp crypto/des/des_enc.c optional crypto @@ -62,6 +75,7 @@ ia64/ia64/autoconf.c standard ia64/ia64/busdma_machdep.c standard ia64/ia64/clock.c standard ia64/ia64/clock_if.m standard +ia64/ia64/context.s standard ia64/ia64/critical.c standard ia64/ia64/db_disasm.c optional ddb ia64/ia64/db_interface.c optional ddb @@ -88,8 +102,8 @@ ia64/ia64/setjmp.s standard ia64/ia64/ssc.c optional ski ia64/ia64/sscdisk.c optional ski ia64/ia64/support.s standard -ia64/ia64/swtch.s standard ia64/ia64/sys_machdep.c standard +ia64/ia64/syscall.s standard ia64/ia64/trap.c standard ia64/ia64/unaligned.c standard ia64/ia64/unwind.c standard diff --git a/sys/conf/options.ia64 b/sys/conf/options.ia64 index 0428336..d035871 100644 --- a/sys/conf/options.ia64 +++ b/sys/conf/options.ia64 @@ -4,7 +4,7 @@ ITANIUM opt_global.h ITANIUM2 opt_global.h -IA32 +IA32 opt_global.h PAGE_SIZE_4K opt_global.h PAGE_SIZE_8K opt_global.h diff --git a/sys/ia64/ia32/ia32_signal.c b/sys/ia64/ia32/ia32_signal.c index 18f4ee2..d9066d4 100644 --- a/sys/ia64/ia32/ia32_signal.c +++ b/sys/ia64/ia32/ia32_signal.c @@ -52,6 +52,10 @@ #include <sys/vnode.h> #include <sys/imgact_elf.h> +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/pcb.h> + #include <vm/vm.h> #include <vm/vm_kern.h> #include <vm/vm_param.h> @@ -64,8 +68,6 @@ #include <i386/include/psl.h> #include <i386/include/segments.h> #include <i386/include/specialreg.h> -#include <machine/frame.h> -#include <machine/md_var.h> static register_t *ia32_copyout_strings(struct image_params *imgp); static void ia32_setregs(struct thread *td, u_long entry, u_long stack, @@ -84,9 +86,9 @@ static char ia32_sigcode[] = { 0x50, /* pushl %eax */ 0xcd, 0x80, /* int $0x80 */ 0xeb, 0xfe, /* 0: jmp 0b */ - 0, 0, 0, 0 + 0 }; -static int ia32_szsigcode = sizeof(ia32_sigcode) & ~3; +static int ia32_szsigcode = sizeof(ia32_sigcode); struct sysentvec ia32_freebsd_sysvec = { SYS_MAXSYSCALL, @@ -105,8 +107,8 @@ struct sysentvec ia32_freebsd_sysvec = { "FreeBSD ELF", elf32_coredump, NULL, - MINSIGSTKSZ, - 4096, + IA32_MINSIGSTKSZ, + IA32_PAGE_SIZE, 0, IA32_USRSTACK, IA32_USRSTACK, @@ -145,8 +147,8 @@ ia32_copyout_strings(struct image_params *imgp) */ arginfo = (struct ia32_ps_strings *)IA32_PS_STRINGS; szsigcode = *(imgp->proc->p_sysent->sv_szsigcode); - destp = (caddr_t)arginfo - szsigcode - SPARE_USRSPACE - - roundup((ARG_MAX - imgp->stringspace), sizeof(char *)); + destp = (caddr_t)arginfo - szsigcode - IA32_USRSPACE - + roundup((ARG_MAX - imgp->stringspace), sizeof(char *)); /* * install sigcode @@ -185,6 +187,7 @@ ia32_copyout_strings(struct image_params *imgp) /* * vectp also becomes our initial stack base */ + vectp = (void*)((uintptr_t)vectp & ~15); stack_base = vectp; stringp = imgp->stringbase; @@ -237,60 +240,45 @@ ia32_copyout_strings(struct image_params *imgp) static void ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) { - struct trapframe *frame = td->td_frame; + struct trapframe *tf = td->td_frame; vm_offset_t gdt, ldt; u_int64_t codesel, datasel, ldtsel; u_int64_t codeseg, dataseg, gdtseg, ldtseg; struct segment_descriptor desc; struct vmspace *vmspace = td->td_proc->p_vmspace; - /* - * Make sure that we restore the entire trapframe after an - * execve. - */ - frame->tf_flags &= ~FRAME_SYSCALL; - - bzero(frame->tf_r, sizeof(frame->tf_r)); - bzero(frame->tf_f, sizeof(frame->tf_f)); - - frame->tf_cr_iip = entry; - frame->tf_cr_ipsr = (IA64_PSR_IC - | IA64_PSR_I - | IA64_PSR_IT - | IA64_PSR_DT - | IA64_PSR_RT - | IA64_PSR_DFH - | IA64_PSR_IS - | IA64_PSR_BN - | IA64_PSR_CPL_USER); - frame->tf_r[FRAME_R12] = stack; + exec_setregs(td, entry, stack, ps_strings); + + /* Non-syscall frames are cleared by exec_setregs() */ + if (tf->tf_flags & FRAME_SYSCALL) { + bzero(&tf->tf_scratch, sizeof(tf->tf_scratch)); + bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp)); + } else + tf->tf_special.ndirty = 0; + + tf->tf_special.psr |= IA64_PSR_IS; + tf->tf_special.sp = stack; + + /* Point the RSE backstore to something harmless. */ + tf->tf_special.bspstore = (IA32_PS_STRINGS - ia32_szsigcode - + IA32_USRSPACE + 15) & ~15; codesel = LSEL(LUCODE_SEL, SEL_UPL); datasel = LSEL(LUDATA_SEL, SEL_UPL); ldtsel = GSEL(GLDT_SEL, SEL_UPL); -#if 1 - frame->tf_r[FRAME_R16] = (datasel << 48) | (datasel << 32) - | (datasel << 16) | datasel; - frame->tf_r[FRAME_R17] = (ldtsel << 32) | (datasel << 16) | codesel; -#else - frame->tf_r[FRAME_R16] = datasel; - frame->tf_r[FRAME_R17] = codesel; - frame->tf_r[FRAME_R18] = datasel; - frame->tf_r[FRAME_R19] = datasel; - frame->tf_r[FRAME_R20] = datasel; - frame->tf_r[FRAME_R21] = datasel; - frame->tf_r[FRAME_R22] = ldtsel; -#endif + /* Setup ia32 segment registers. */ + tf->tf_scratch.gr16 = (datasel << 48) | (datasel << 32) | + (datasel << 16) | datasel; + tf->tf_scratch.gr17 = (ldtsel << 32) | (datasel << 16) | codesel; /* * Build the GDT and LDT. */ gdt = IA32_USRSTACK; - vm_map_find(&vmspace->vm_map, 0, 0, - &gdt, PAGE_SIZE, 0, - VM_PROT_ALL, VM_PROT_ALL, 0); - ldt = gdt + 4096; + vm_map_find(&vmspace->vm_map, 0, 0, &gdt, IA32_PAGE_SIZE << 1, 0, + VM_PROT_ALL, VM_PROT_ALL, 0); + ldt = gdt + IA32_PAGE_SIZE; desc.sd_lolimit = 8*NLDT-1; desc.sd_lobase = ldt & 0xffffff; @@ -330,12 +318,13 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) + (1L << 59) /* present */ + (1L << 62) /* 32 bits */ + (1L << 63); /* page granularity */ - ia64_set_csd(codeseg); - ia64_set_ssd(dataseg); - frame->tf_r[FRAME_R24] = dataseg; /* ESD */ - frame->tf_r[FRAME_R27] = dataseg; /* DSD */ - frame->tf_r[FRAME_R28] = dataseg; /* FSD */ - frame->tf_r[FRAME_R29] = dataseg; /* GSD */ + + tf->tf_scratch.csd = codeseg; + tf->tf_scratch.ssd = dataseg; + tf->tf_scratch.gr24 = dataseg; /* ESD */ + tf->tf_scratch.gr27 = dataseg; /* DSD */ + tf->tf_scratch.gr28 = dataseg; /* FSD */ + tf->tf_scratch.gr29 = dataseg; /* GSD */ gdtseg = gdt /* base */ + ((8L*NGDT - 1) << 32) /* limit */ @@ -351,13 +340,16 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) + (1L << 59) /* present */ + (0L << 62) /* 16 bits */ + (0L << 63); /* byte granularity */ - frame->tf_r[FRAME_R30] = ldtseg; /* LDTD */ - frame->tf_r[FRAME_R31] = gdtseg; /* GDTD */ + tf->tf_scratch.gr30 = ldtseg; /* LDTD */ + tf->tf_scratch.gr31 = gdtseg; /* GDTD */ + + /* Set ia32 control registers on this processor. */ + ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM | CR4_FXSR) << 32)); ia64_set_eflag(PSL_USER); /* PS_STRINGS value for BSD/OS binaries. It is 0 for non-BSD/OS. */ - frame->tf_r[FRAME_R11] = IA32_PS_STRINGS; + tf->tf_scratch.gr11 = IA32_PS_STRINGS; /* * XXX - Linux emulator @@ -366,3 +358,27 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) */ td->td_retval[1] = 0; } + +void +ia32_restorectx(struct pcb *pcb) +{ + + ia64_set_cflg(pcb->pcb_ia32_cflg); + ia64_set_eflag(pcb->pcb_ia32_eflag); + ia64_set_fcr(pcb->pcb_ia32_fcr); + ia64_set_fdr(pcb->pcb_ia32_fdr); + ia64_set_fir(pcb->pcb_ia32_fir); + ia64_set_fsr(pcb->pcb_ia32_fsr); +} + +void +ia32_savectx(struct pcb *pcb) +{ + + pcb->pcb_ia32_cflg = ia64_get_cflg(); + pcb->pcb_ia32_eflag = ia64_get_eflag(); + pcb->pcb_ia32_fcr = ia64_get_fcr(); + pcb->pcb_ia32_fdr = ia64_get_fdr(); + pcb->pcb_ia32_fir = ia64_get_fir(); + pcb->pcb_ia32_fsr = ia64_get_fsr(); +} diff --git a/sys/ia64/ia32/ia32_sysvec.c b/sys/ia64/ia32/ia32_sysvec.c index 18f4ee2..d9066d4 100644 --- a/sys/ia64/ia32/ia32_sysvec.c +++ b/sys/ia64/ia32/ia32_sysvec.c @@ -52,6 +52,10 @@ #include <sys/vnode.h> #include <sys/imgact_elf.h> +#include <machine/frame.h> +#include <machine/md_var.h> +#include <machine/pcb.h> + #include <vm/vm.h> #include <vm/vm_kern.h> #include <vm/vm_param.h> @@ -64,8 +68,6 @@ #include <i386/include/psl.h> #include <i386/include/segments.h> #include <i386/include/specialreg.h> -#include <machine/frame.h> -#include <machine/md_var.h> static register_t *ia32_copyout_strings(struct image_params *imgp); static void ia32_setregs(struct thread *td, u_long entry, u_long stack, @@ -84,9 +86,9 @@ static char ia32_sigcode[] = { 0x50, /* pushl %eax */ 0xcd, 0x80, /* int $0x80 */ 0xeb, 0xfe, /* 0: jmp 0b */ - 0, 0, 0, 0 + 0 }; -static int ia32_szsigcode = sizeof(ia32_sigcode) & ~3; +static int ia32_szsigcode = sizeof(ia32_sigcode); struct sysentvec ia32_freebsd_sysvec = { SYS_MAXSYSCALL, @@ -105,8 +107,8 @@ struct sysentvec ia32_freebsd_sysvec = { "FreeBSD ELF", elf32_coredump, NULL, - MINSIGSTKSZ, - 4096, + IA32_MINSIGSTKSZ, + IA32_PAGE_SIZE, 0, IA32_USRSTACK, IA32_USRSTACK, @@ -145,8 +147,8 @@ ia32_copyout_strings(struct image_params *imgp) */ arginfo = (struct ia32_ps_strings *)IA32_PS_STRINGS; szsigcode = *(imgp->proc->p_sysent->sv_szsigcode); - destp = (caddr_t)arginfo - szsigcode - SPARE_USRSPACE - - roundup((ARG_MAX - imgp->stringspace), sizeof(char *)); + destp = (caddr_t)arginfo - szsigcode - IA32_USRSPACE - + roundup((ARG_MAX - imgp->stringspace), sizeof(char *)); /* * install sigcode @@ -185,6 +187,7 @@ ia32_copyout_strings(struct image_params *imgp) /* * vectp also becomes our initial stack base */ + vectp = (void*)((uintptr_t)vectp & ~15); stack_base = vectp; stringp = imgp->stringbase; @@ -237,60 +240,45 @@ ia32_copyout_strings(struct image_params *imgp) static void ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) { - struct trapframe *frame = td->td_frame; + struct trapframe *tf = td->td_frame; vm_offset_t gdt, ldt; u_int64_t codesel, datasel, ldtsel; u_int64_t codeseg, dataseg, gdtseg, ldtseg; struct segment_descriptor desc; struct vmspace *vmspace = td->td_proc->p_vmspace; - /* - * Make sure that we restore the entire trapframe after an - * execve. - */ - frame->tf_flags &= ~FRAME_SYSCALL; - - bzero(frame->tf_r, sizeof(frame->tf_r)); - bzero(frame->tf_f, sizeof(frame->tf_f)); - - frame->tf_cr_iip = entry; - frame->tf_cr_ipsr = (IA64_PSR_IC - | IA64_PSR_I - | IA64_PSR_IT - | IA64_PSR_DT - | IA64_PSR_RT - | IA64_PSR_DFH - | IA64_PSR_IS - | IA64_PSR_BN - | IA64_PSR_CPL_USER); - frame->tf_r[FRAME_R12] = stack; + exec_setregs(td, entry, stack, ps_strings); + + /* Non-syscall frames are cleared by exec_setregs() */ + if (tf->tf_flags & FRAME_SYSCALL) { + bzero(&tf->tf_scratch, sizeof(tf->tf_scratch)); + bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp)); + } else + tf->tf_special.ndirty = 0; + + tf->tf_special.psr |= IA64_PSR_IS; + tf->tf_special.sp = stack; + + /* Point the RSE backstore to something harmless. */ + tf->tf_special.bspstore = (IA32_PS_STRINGS - ia32_szsigcode - + IA32_USRSPACE + 15) & ~15; codesel = LSEL(LUCODE_SEL, SEL_UPL); datasel = LSEL(LUDATA_SEL, SEL_UPL); ldtsel = GSEL(GLDT_SEL, SEL_UPL); -#if 1 - frame->tf_r[FRAME_R16] = (datasel << 48) | (datasel << 32) - | (datasel << 16) | datasel; - frame->tf_r[FRAME_R17] = (ldtsel << 32) | (datasel << 16) | codesel; -#else - frame->tf_r[FRAME_R16] = datasel; - frame->tf_r[FRAME_R17] = codesel; - frame->tf_r[FRAME_R18] = datasel; - frame->tf_r[FRAME_R19] = datasel; - frame->tf_r[FRAME_R20] = datasel; - frame->tf_r[FRAME_R21] = datasel; - frame->tf_r[FRAME_R22] = ldtsel; -#endif + /* Setup ia32 segment registers. */ + tf->tf_scratch.gr16 = (datasel << 48) | (datasel << 32) | + (datasel << 16) | datasel; + tf->tf_scratch.gr17 = (ldtsel << 32) | (datasel << 16) | codesel; /* * Build the GDT and LDT. */ gdt = IA32_USRSTACK; - vm_map_find(&vmspace->vm_map, 0, 0, - &gdt, PAGE_SIZE, 0, - VM_PROT_ALL, VM_PROT_ALL, 0); - ldt = gdt + 4096; + vm_map_find(&vmspace->vm_map, 0, 0, &gdt, IA32_PAGE_SIZE << 1, 0, + VM_PROT_ALL, VM_PROT_ALL, 0); + ldt = gdt + IA32_PAGE_SIZE; desc.sd_lolimit = 8*NLDT-1; desc.sd_lobase = ldt & 0xffffff; @@ -330,12 +318,13 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) + (1L << 59) /* present */ + (1L << 62) /* 32 bits */ + (1L << 63); /* page granularity */ - ia64_set_csd(codeseg); - ia64_set_ssd(dataseg); - frame->tf_r[FRAME_R24] = dataseg; /* ESD */ - frame->tf_r[FRAME_R27] = dataseg; /* DSD */ - frame->tf_r[FRAME_R28] = dataseg; /* FSD */ - frame->tf_r[FRAME_R29] = dataseg; /* GSD */ + + tf->tf_scratch.csd = codeseg; + tf->tf_scratch.ssd = dataseg; + tf->tf_scratch.gr24 = dataseg; /* ESD */ + tf->tf_scratch.gr27 = dataseg; /* DSD */ + tf->tf_scratch.gr28 = dataseg; /* FSD */ + tf->tf_scratch.gr29 = dataseg; /* GSD */ gdtseg = gdt /* base */ + ((8L*NGDT - 1) << 32) /* limit */ @@ -351,13 +340,16 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) + (1L << 59) /* present */ + (0L << 62) /* 16 bits */ + (0L << 63); /* byte granularity */ - frame->tf_r[FRAME_R30] = ldtseg; /* LDTD */ - frame->tf_r[FRAME_R31] = gdtseg; /* GDTD */ + tf->tf_scratch.gr30 = ldtseg; /* LDTD */ + tf->tf_scratch.gr31 = gdtseg; /* GDTD */ + + /* Set ia32 control registers on this processor. */ + ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM | CR4_FXSR) << 32)); ia64_set_eflag(PSL_USER); /* PS_STRINGS value for BSD/OS binaries. It is 0 for non-BSD/OS. */ - frame->tf_r[FRAME_R11] = IA32_PS_STRINGS; + tf->tf_scratch.gr11 = IA32_PS_STRINGS; /* * XXX - Linux emulator @@ -366,3 +358,27 @@ ia32_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) */ td->td_retval[1] = 0; } + +void +ia32_restorectx(struct pcb *pcb) +{ + + ia64_set_cflg(pcb->pcb_ia32_cflg); + ia64_set_eflag(pcb->pcb_ia32_eflag); + ia64_set_fcr(pcb->pcb_ia32_fcr); + ia64_set_fdr(pcb->pcb_ia32_fdr); + ia64_set_fir(pcb->pcb_ia32_fir); + ia64_set_fsr(pcb->pcb_ia32_fsr); +} + +void +ia32_savectx(struct pcb *pcb) +{ + + pcb->pcb_ia32_cflg = ia64_get_cflg(); + pcb->pcb_ia32_eflag = ia64_get_eflag(); + pcb->pcb_ia32_fcr = ia64_get_fcr(); + pcb->pcb_ia32_fdr = ia64_get_fdr(); + pcb->pcb_ia32_fir = ia64_get_fir(); + pcb->pcb_ia32_fsr = ia64_get_fsr(); +} diff --git a/sys/ia64/ia32/ia32_util.h b/sys/ia64/ia32/ia32_util.h index 23f2aba..a0b21b0 100644 --- a/sys/ia64/ia32/ia32_util.h +++ b/sys/ia64/ia32/ia32_util.h @@ -44,8 +44,11 @@ struct ia32_ps_strings { int ps_nenvstr; /* the number of environment strings */ }; -#define IA32_USRSTACK (4L*1024*1024*1024 - PAGE_SIZE) -#define IA32_PS_STRINGS (IA32_USRSTACK - sizeof(struct ia32_ps_strings)) +#define IA32_MINSIGSTKSZ 2048 +#define IA32_PAGE_SIZE 4096 +#define IA32_USRSTACK (2L*1024*1024*1024 - IA32_PAGE_SIZE*2) +#define IA32_PS_STRINGS (IA32_USRSTACK - sizeof(struct ia32_ps_strings)) +#define IA32_USRSPACE IA32_PAGE_SIZE static __inline caddr_t stackgap_init(void); static __inline void *stackgap_alloc(caddr_t *, size_t); @@ -54,7 +57,7 @@ static __inline caddr_t stackgap_init() { #define szsigcode (*(curproc->p_sysent->sv_szsigcode)) - return (caddr_t)(((caddr_t)IA32_PS_STRINGS) - szsigcode - SPARE_USRSPACE); + return (((caddr_t)IA32_PS_STRINGS) - szsigcode - IA32_USRSPACE); #undef szsigcode } diff --git a/sys/ia64/ia64/db_interface.c b/sys/ia64/ia64/db_interface.c index 34ac65d..3ddece0 100644 --- a/sys/ia64/ia64/db_interface.c +++ b/sys/ia64/ia64/db_interface.c @@ -51,7 +51,6 @@ #include <vm/vm.h> #include <machine/inst.h> -#include <machine/rse.h> #include <machine/db_machdep.h> #include <machine/mutex.h> @@ -70,73 +69,61 @@ extern void gdb_handle_exception(db_regs_t *, int); int db_active; db_regs_t ddb_regs; -static u_int64_t zero; static int db_get_rse_reg(struct db_variable *vp, db_expr_t *valuep, int op); -static int db_get_pc_reg(struct db_variable *vp, db_expr_t *valuep, int op); +static int db_get_ip_reg(struct db_variable *vp, db_expr_t *valuep, int op); struct db_variable db_regs[] = { /* Misc control/app registers */ -#define DB_MISC_REGS 15 /* make sure this is correct */ - - {"pc", (db_expr_t*) 0, db_get_pc_reg}, - {"ip", (db_expr_t*) &ddb_regs.tf_cr_iip, FCN_NULL}, - {"psr", (db_expr_t*) &ddb_regs.tf_cr_ipsr, FCN_NULL}, - {"cr.isr", (db_expr_t*) &ddb_regs.tf_cr_isr, FCN_NULL}, - {"cr.ifa", (db_expr_t*) &ddb_regs.tf_cr_ifa, FCN_NULL}, - {"pr", (db_expr_t*) &ddb_regs.tf_pr, FCN_NULL}, - {"ar.rsc", (db_expr_t*) &ddb_regs.tf_ar_rsc, FCN_NULL}, - {"ar.pfs", (db_expr_t*) &ddb_regs.tf_ar_pfs, FCN_NULL}, - {"cr.ifs", (db_expr_t*) &ddb_regs.tf_cr_ifs, FCN_NULL}, - {"ar.bspstore", (db_expr_t*) &ddb_regs.tf_ar_bspstore, FCN_NULL}, - {"ar.rnat", (db_expr_t*) &ddb_regs.tf_ar_rnat, FCN_NULL}, - {"ndirty", (db_expr_t*) &ddb_regs.tf_ndirty, FCN_NULL}, - {"ar.unat", (db_expr_t*) &ddb_regs.tf_ar_unat, FCN_NULL}, - {"ar.ccv", (db_expr_t*) &ddb_regs.tf_ar_ccv, FCN_NULL}, - {"ar.fpsr", (db_expr_t*) &ddb_regs.tf_ar_fpsr, FCN_NULL}, +#define DB_MISC_REGS 13 /* make sure this is correct */ + + {"ip", NULL, db_get_ip_reg}, + {"psr", (db_expr_t*) &ddb_regs.tf_special.psr, FCN_NULL}, + {"cr.isr", (db_expr_t*) &ddb_regs.tf_special.isr, FCN_NULL}, + {"cr.ifa", (db_expr_t*) &ddb_regs.tf_special.ifa, FCN_NULL}, + {"pr", (db_expr_t*) &ddb_regs.tf_special.pr, FCN_NULL}, + {"ar.rsc", (db_expr_t*) &ddb_regs.tf_special.rsc, FCN_NULL}, + {"ar.pfs", (db_expr_t*) &ddb_regs.tf_special.pfs, FCN_NULL}, + {"cr.ifs", (db_expr_t*) &ddb_regs.tf_special.cfm, FCN_NULL}, + {"ar.bspstore", (db_expr_t*) &ddb_regs.tf_special.bspstore, FCN_NULL}, + {"ndirty", (db_expr_t*) &ddb_regs.tf_special.ndirty, FCN_NULL}, + {"ar.rnat", (db_expr_t*) &ddb_regs.tf_special.rnat, FCN_NULL}, + {"ar.unat", (db_expr_t*) &ddb_regs.tf_special.unat, FCN_NULL}, + {"ar.fpsr", (db_expr_t*) &ddb_regs.tf_special.fpsr, FCN_NULL}, /* Branch registers */ - {"rp", (db_expr_t*) &ddb_regs.tf_b[0], FCN_NULL}, - {"b1", (db_expr_t*) &ddb_regs.tf_b[1], FCN_NULL}, - {"b2", (db_expr_t*) &ddb_regs.tf_b[2], FCN_NULL}, - {"b3", (db_expr_t*) &ddb_regs.tf_b[3], FCN_NULL}, - {"b4", (db_expr_t*) &ddb_regs.tf_b[4], FCN_NULL}, - {"b5", (db_expr_t*) &ddb_regs.tf_b[5], FCN_NULL}, - {"b6", (db_expr_t*) &ddb_regs.tf_b[6], FCN_NULL}, - {"b7", (db_expr_t*) &ddb_regs.tf_b[7], FCN_NULL}, + {"rp", (db_expr_t*) &ddb_regs.tf_special.rp, FCN_NULL}, + /* b1, b2, b3, b4, b5 are preserved */ + {"b6", (db_expr_t*) &ddb_regs.tf_scratch.br6, FCN_NULL}, + {"b7", (db_expr_t*) &ddb_regs.tf_scratch.br7, FCN_NULL}, /* Static registers */ - {"r0", (db_expr_t*) &zero, FCN_NULL}, - {"gp", (db_expr_t*) &ddb_regs.tf_r[FRAME_R1], FCN_NULL}, - {"r2", (db_expr_t*) &ddb_regs.tf_r[FRAME_R2], FCN_NULL}, - {"r3", (db_expr_t*) &ddb_regs.tf_r[FRAME_R3], FCN_NULL}, - {"r4", (db_expr_t*) &ddb_regs.tf_r[FRAME_R4], FCN_NULL}, - {"r5", (db_expr_t*) &ddb_regs.tf_r[FRAME_R5], FCN_NULL}, - {"r6", (db_expr_t*) &ddb_regs.tf_r[FRAME_R6], FCN_NULL}, - {"r7", (db_expr_t*) &ddb_regs.tf_r[FRAME_R7], FCN_NULL}, - {"r8", (db_expr_t*) &ddb_regs.tf_r[FRAME_R8], FCN_NULL}, - {"r9", (db_expr_t*) &ddb_regs.tf_r[FRAME_R9], FCN_NULL}, - {"r10", (db_expr_t*) &ddb_regs.tf_r[FRAME_R10], FCN_NULL}, - {"r11", (db_expr_t*) &ddb_regs.tf_r[FRAME_R11], FCN_NULL}, - {"sp", (db_expr_t*) &ddb_regs.tf_r[FRAME_R12], FCN_NULL}, - {"r13", (db_expr_t*) &ddb_regs.tf_r[FRAME_R13], FCN_NULL}, - {"r14", (db_expr_t*) &ddb_regs.tf_r[FRAME_R14], FCN_NULL}, - {"r15", (db_expr_t*) &ddb_regs.tf_r[FRAME_R15], FCN_NULL}, - {"r16", (db_expr_t*) &ddb_regs.tf_r[FRAME_R16], FCN_NULL}, - {"r17", (db_expr_t*) &ddb_regs.tf_r[FRAME_R17], FCN_NULL}, - {"r18", (db_expr_t*) &ddb_regs.tf_r[FRAME_R18], FCN_NULL}, - {"r19", (db_expr_t*) &ddb_regs.tf_r[FRAME_R19], FCN_NULL}, - {"r20", (db_expr_t*) &ddb_regs.tf_r[FRAME_R20], FCN_NULL}, - {"r21", (db_expr_t*) &ddb_regs.tf_r[FRAME_R21], FCN_NULL}, - {"r22", (db_expr_t*) &ddb_regs.tf_r[FRAME_R22], FCN_NULL}, - {"r23", (db_expr_t*) &ddb_regs.tf_r[FRAME_R23], FCN_NULL}, - {"r24", (db_expr_t*) &ddb_regs.tf_r[FRAME_R24], FCN_NULL}, - {"r25", (db_expr_t*) &ddb_regs.tf_r[FRAME_R25], FCN_NULL}, - {"r26", (db_expr_t*) &ddb_regs.tf_r[FRAME_R26], FCN_NULL}, - {"r27", (db_expr_t*) &ddb_regs.tf_r[FRAME_R27], FCN_NULL}, - {"r28", (db_expr_t*) &ddb_regs.tf_r[FRAME_R28], FCN_NULL}, - {"r29", (db_expr_t*) &ddb_regs.tf_r[FRAME_R29], FCN_NULL}, - {"r30", (db_expr_t*) &ddb_regs.tf_r[FRAME_R30], FCN_NULL}, - {"r31", (db_expr_t*) &ddb_regs.tf_r[FRAME_R31], FCN_NULL}, + {"gp", (db_expr_t*) &ddb_regs.tf_special.gp, FCN_NULL}, + {"r2", (db_expr_t*) &ddb_regs.tf_scratch.gr2, FCN_NULL}, + {"r3", (db_expr_t*) &ddb_regs.tf_scratch.gr3, FCN_NULL}, + {"r8", (db_expr_t*) &ddb_regs.tf_scratch.gr8, FCN_NULL}, + {"r9", (db_expr_t*) &ddb_regs.tf_scratch.gr9, FCN_NULL}, + {"r10", (db_expr_t*) &ddb_regs.tf_scratch.gr10, FCN_NULL}, + {"r11", (db_expr_t*) &ddb_regs.tf_scratch.gr11, FCN_NULL}, + {"sp", (db_expr_t*) &ddb_regs.tf_special.sp, FCN_NULL}, + {"tp", (db_expr_t*) &ddb_regs.tf_special.tp, FCN_NULL}, + {"r14", (db_expr_t*) &ddb_regs.tf_scratch.gr14, FCN_NULL}, + {"r15", (db_expr_t*) &ddb_regs.tf_scratch.gr15, FCN_NULL}, + {"r16", (db_expr_t*) &ddb_regs.tf_scratch.gr16, FCN_NULL}, + {"r17", (db_expr_t*) &ddb_regs.tf_scratch.gr17, FCN_NULL}, + {"r18", (db_expr_t*) &ddb_regs.tf_scratch.gr18, FCN_NULL}, + {"r19", (db_expr_t*) &ddb_regs.tf_scratch.gr19, FCN_NULL}, + {"r20", (db_expr_t*) &ddb_regs.tf_scratch.gr20, FCN_NULL}, + {"r21", (db_expr_t*) &ddb_regs.tf_scratch.gr21, FCN_NULL}, + {"r22", (db_expr_t*) &ddb_regs.tf_scratch.gr22, FCN_NULL}, + {"r23", (db_expr_t*) &ddb_regs.tf_scratch.gr23, FCN_NULL}, + {"r24", (db_expr_t*) &ddb_regs.tf_scratch.gr24, FCN_NULL}, + {"r25", (db_expr_t*) &ddb_regs.tf_scratch.gr25, FCN_NULL}, + {"r26", (db_expr_t*) &ddb_regs.tf_scratch.gr26, FCN_NULL}, + {"r27", (db_expr_t*) &ddb_regs.tf_scratch.gr27, FCN_NULL}, + {"r28", (db_expr_t*) &ddb_regs.tf_scratch.gr28, FCN_NULL}, + {"r29", (db_expr_t*) &ddb_regs.tf_scratch.gr29, FCN_NULL}, + {"r30", (db_expr_t*) &ddb_regs.tf_scratch.gr30, FCN_NULL}, + {"r31", (db_expr_t*) &ddb_regs.tf_scratch.gr31, FCN_NULL}, /* Stacked registers */ {"r32", (db_expr_t*) 32, db_get_rse_reg}, @@ -241,28 +228,32 @@ struct db_variable *db_eregs = db_regs + sizeof(db_regs)/sizeof(db_regs[0]); static int db_get_rse_reg(struct db_variable *vp, db_expr_t *valuep, int op) { - int sof = ddb_regs.tf_cr_ifs & 0x7f; - int regno = (db_expr_t) vp->valuep; - u_int64_t *bsp = (u_int64_t *) (ddb_regs.tf_ar_bspstore + ddb_regs.tf_ndirty); u_int64_t *reg; + uint64_t bsp; + int nats, regno, sof; - if (regno - 32 >= sof) { - if (op == DB_VAR_GET) - *valuep = 0xdeadbeefdeadbeef; - } else { - bsp = ia64_rse_previous_frame(bsp, sof); - reg = ia64_rse_register_address(bsp, regno); + bsp = ddb_regs.tf_special.bspstore + ddb_regs.tf_special.ndirty; + regno = (db_expr_t)vp->valuep - 32; + sof = (int)(ddb_regs.tf_special.cfm & 0x7f); + nats = (sof - regno + 63 - ((int)(bsp >> 3) & 0x3f)) / 63; + + reg = (void*)(bsp - ((sof - regno + nats) << 3)); + + if (regno < sof) { if (op == DB_VAR_GET) *valuep = *reg; else *reg = *valuep; + } else { + if (op == DB_VAR_GET) + *valuep = 0xdeadbeefdeadbeef; } - return 0; + return (0); } static int -db_get_pc_reg(struct db_variable *vp, db_expr_t *valuep, int op) +db_get_ip_reg(struct db_variable *vp, db_expr_t *valuep, int op) { /* Read only */ if (op == DB_VAR_GET) @@ -353,8 +344,8 @@ kdb_trap(int vector, struct trapframe *regs) /* * XXX pretend that registers outside the current frame don't exist. */ - db_eregs = db_regs + DB_MISC_REGS + 8 + 32 - + (ddb_regs.tf_cr_ifs & 0x7f); + db_eregs = db_regs + DB_MISC_REGS + 3 + 27 + + (ddb_regs.tf_special.cfm & 0x7f); __asm __volatile("flushrs"); /* so we can look at them */ @@ -412,13 +403,13 @@ db_read_bytes(addr, size, data) register size_t size; register char *data; { - register char *src; db_nofault = &db_jmpbuf; - src = (char *)addr; - while (size-- > 0) - *data++ = *src++; + if (addr < VM_MAX_ADDRESS) + copyin((char *)addr, data, size); + else + bcopy((char *)addr, data, size); db_nofault = 0; } @@ -432,13 +423,13 @@ db_write_bytes(addr, size, data) register size_t size; register char *data; { - register char *dst; db_nofault = &db_jmpbuf; - dst = (char *)addr; - while (size-- > 0) - *dst++ = *data++; + if (addr < VM_MAX_ADDRESS) + copyout(data, (char *)addr, size); + else + bcopy(data, (char *)addr, size); db_nofault = 0; } @@ -455,30 +446,36 @@ db_register_value(regs, regno) db_regs_t *regs; int regno; { - - if (regno > 127 || regno < 0) { - db_printf(" **** STRANGE REGISTER NUMBER %d **** ", regno); - return (0); - } + uint64_t *rsp; + uint64_t bsp; + int nats, sof; if (regno == 0) return (0); - - if (regno < 32) { - return (regs->tf_r[regno - 1]); - } else { - int sof = ddb_regs.tf_cr_ifs & 0x7f; - u_int64_t *bsp = (u_int64_t *) (ddb_regs.tf_ar_bspstore + ddb_regs.tf_ndirty); - u_int64_t *reg; - - if (regno - 32 >= sof) { - return 0xdeadbeefdeadbeef; - } else { - bsp = ia64_rse_previous_frame(bsp, sof); - reg = ia64_rse_register_address(bsp, regno); - return *reg; - } + if (regno == 1) + return (regs->tf_special.gp); + if (regno >= 2 && regno <= 3) + return ((®s->tf_scratch.gr2)[regno - 2]); + if (regno >= 8 && regno <= 11) + return ((®s->tf_scratch.gr8)[regno - 8]); + if (regno == 12) + return (regs->tf_special.sp); + if (regno == 13) + return (regs->tf_special.tp); + if (regno >= 14 && regno <= 31) + return ((®s->tf_scratch.gr14)[regno - 14]); + + sof = (int)(regs->tf_special.cfm & 0x7f); + if (regno >= 32 && regno < sof + 32) { + bsp = regs->tf_special.bspstore + regs->tf_special.ndirty; + regno -= 32; + nats = (sof - regno + 63 - ((int)(bsp >> 3) & 0x3f)) / 63; + rsp = (void*)(bsp - ((sof - regno + nats) << 3)); + return (*rsp); } + + db_printf(" **** STRANGE REGISTER NUMBER %d **** ", regno); + return (0); } void @@ -539,10 +536,10 @@ db_skip_breakpoint(void) /* * Skip past the break instruction. */ - ddb_regs.tf_cr_ipsr += IA64_PSR_RI_1; - if ((ddb_regs.tf_cr_ipsr & IA64_PSR_RI) > IA64_PSR_RI_2) { - ddb_regs.tf_cr_ipsr &= ~IA64_PSR_RI; - ddb_regs.tf_cr_iip += 16; + ddb_regs.tf_special.psr += IA64_PSR_RI_1; + if ((ddb_regs.tf_special.psr & IA64_PSR_RI) > IA64_PSR_RI_2) { + ddb_regs.tf_special.psr &= ~IA64_PSR_RI; + ddb_regs.tf_special.iip += 16; } } diff --git a/sys/ia64/ia64/db_trace.c b/sys/ia64/ia64/db_trace.c index 7c79d06..f093561 100644 --- a/sys/ia64/ia64/db_trace.c +++ b/sys/ia64/ia64/db_trace.c @@ -31,7 +31,6 @@ #include <machine/inst.h> #include <machine/db_machdep.h> #include <machine/unwind.h> -#include <machine/rse.h> #include <machine/vmparam.h> #include <ddb/ddb.h> @@ -45,74 +44,61 @@ int db_md_set_watchpoint(db_expr_t addr, db_expr_t size); int db_md_clr_watchpoint(db_expr_t addr, db_expr_t size); void db_md_list_watchpoints(void); -extern char ia64_vector_table[], do_syscall[], do_syscall_end[]; - void -db_stack_trace_cmd(db_expr_t addr, boolean_t have_addr, db_expr_t count, char *modif) +db_stack_trace_cmd(db_expr_t addr, boolean_t have_addr, db_expr_t count, + char *modif) { - struct ia64_unwind_state *us; - - if (count == -1) - count = 65535; - - if (!have_addr) { - us = ia64_create_unwind_state(&ddb_regs); - } else { - return; /* XXX */ - } - - if (!us) { - db_printf("db_stack_trace_cmd: can't create unwind state\n"); - return; - } + struct unw_regstate rs; + const char *name; + db_expr_t offset; + uint64_t bsp, cfm, ip, pfs, reg; + c_db_sym_t sym; + int args, error, i; + + error = unw_create(&rs, &ddb_regs); + while (!error && count--) { + error = unw_get_cfm(&rs, &cfm); + if (!error) + error = unw_get_bsp(&rs, &bsp); + if (!error) + error = unw_get_ip(&rs, &ip); + if (error) + break; - while (count--) { - const char * name; - db_expr_t ip; - db_expr_t offset; - c_db_sym_t sym; - int cfm, sof, sol, nargs, i; - u_int64_t *bsp; - u_int64_t *p, reg; - - ip = ia64_unwind_state_get_ip(us); - cfm = ia64_unwind_state_get_cfm(us); - bsp = ia64_unwind_state_get_bsp(us); - sof = cfm & 0x7f; - sol = (cfm >> 7) & 0x7f; + args = (cfm >> 7) & 0x7f; + if (args > 8) + args = 8; + + error = unw_step(&rs); + if (!error) { + error = unw_get_cfm(&rs, &pfs); + if (!error) { + i = (pfs & 0x7f) - ((pfs >> 7) & 0x7f); + if (args > i) + args = i; + } + } sym = db_search_symbol(ip, DB_STGY_ANY, &offset); db_symbol_values(sym, &name, NULL); - db_printf("%s(", name); - - nargs = sof - sol; - if (nargs > 8) - nargs = 8; - if (bsp >= (u_int64_t *)IA64_RR_BASE(5)) { - for (i = 0; i < nargs; i++) { - p = ia64_rse_register_address(bsp, 32 + i); - db_read_bytes((vm_offset_t) p, sizeof(reg), - (caddr_t) ®); + if (bsp >= IA64_RR_BASE(5)) { + for (i = 0; i < args; i++) { + if ((bsp & 0x1ff) == 0x1f8) + bsp += 8; + db_read_bytes(bsp, sizeof(reg), (void*)®); if (i > 0) db_printf(", "); db_printf("0x%lx", reg); + bsp += 8; } - } + } else + db_printf("..."); db_printf(") at "); db_printsym(ip, DB_STGY_PROC); db_printf("\n"); - - if (ia64_unwind_state_previous_frame(us)) - break; - - ip = ia64_unwind_state_get_ip(us); - if (!ip) - break; } - - ia64_free_unwind_state(us); } void @@ -143,4 +129,3 @@ db_md_list_watchpoints() { return; } - diff --git a/sys/ia64/ia64/elf_machdep.c b/sys/ia64/ia64/elf_machdep.c index b3c7d90..6cecfb0 100644 --- a/sys/ia64/ia64/elf_machdep.c +++ b/sys/ia64/ia64/elf_machdep.c @@ -59,8 +59,8 @@ struct sysentvec elf64_freebsd_sysvec = { NULL, __elfN(freebsd_fixup), sendsig, - sigcode, - &szsigcode, + NULL, /* sigcode */ + NULL, /* &szsigcode */ NULL, "FreeBSD ELF64", __elfN(coredump), @@ -262,7 +262,7 @@ elf_cpu_load_file(linker_file_t lf) if (ph->p_type == PT_IA_64_UNWIND) { vaddr = ph->p_vaddr + reloc; - ia64_add_unwind_table((vm_offset_t)lf->address, vaddr, + unw_table_add((vm_offset_t)lf->address, vaddr, vaddr + ph->p_memsz); } ++ph; @@ -275,6 +275,6 @@ int elf_cpu_unload_file(linker_file_t lf) { - ia64_delete_unwind_table((vm_offset_t)lf->address); + unw_table_remove((vm_offset_t)lf->address); return (0); } diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S index 9e89aaa..947df2d 100644 --- a/sys/ia64/ia64/exception.S +++ b/sys/ia64/ia64/exception.S @@ -1,4 +1,5 @@ /*- + * Copyright (c) 2003 Marcel Moolenaar * Copyright (c) 2000 Doug Rabson * All rights reserved. * @@ -31,12 +32,541 @@ #include <assym.s> /* - * ar.k7 = curthread - * ar.k6 = ksp - * ar.k5 = kbsp - * ar.k4 = pcpup + * ar.k7 = kernel memory stack + * ar.k6 = kernel register stack + * ar.k5 = EPC gateway page + * ar.k4 = PCPU data */ + .text + +/* + * exception_save: save interrupted state + * + * Arguments: + * r16 address of bundle that contains the branch. The + * return address will be the next bundle. + * r17 the value to save as ifa in the trapframe. This + * normally is cr.ifa, but some interruptions set + * set cr.iim and not cr.ifa. + * + * Returns: + * p15 interrupted from user stack + * p14 interrupted from kernel stack + * p13 interrupted from user backing store + * p12 interrupted from kernel backing store + * p11 interrupts were enabled + * p10 interrupts were disabled + */ +ENTRY(exception_save, 0) +{ .mii + mov r20=ar.unat + extr.u r31=sp,61,3 + mov r18=pr + ;; +} +{ .mmi + cmp.le p14,p15=5,r31 + ;; +(p15) mov r23=ar.k7 // kernel memory stack +(p14) mov r23=sp + ;; +} +{ .mii + mov r21=ar.rsc + add r30=-SIZEOF_TRAPFRAME,r23 + ;; + dep r30=0,r30,0,10 + ;; +} +{ .mmi + mov ar.rsc=0 + sub r19=r23,r30 + add r31=8,r30 + ;; +} +{ .mlx + mov r22=cr.iip + movl r26=exception_save_restart + ;; +} + + /* + * We have a 1KB aligned trapframe, pointed to by sp. If we write + * to the trapframe, we may trigger a data nested TLB fault. By + * aligning the trapframe on a 1KB boundary, we guarantee that if + * we get a data nested TLB fault, it will be on the very first + * write. Since the data nested TLB fault does not preserve any + * state, we have to be careful what we clobber. Consequently, we + * have to be careful what we use here. Below a list of registers + * that are currently alive: + * r16,r17=arguments + * r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS + * r26=restart point + * r30,r31=trapframe pointers + * p14,p15=memory stack switch + */ +exception_save_restart: +{ .mmi + st8 [r30]=r19,16 // length + st8 [r31]=r0,16 // flags + add r19=16,r19 + ;; +} +{ .mmi + st8.spill [r30]=sp,16 // sp + st8 [r31]=r20,16 // unat + sub sp=r23,r19 + ;; +} +{ .mmi + mov r19=ar.rnat + mov r20=ar.bspstore + mov r23=rp + ;; +} + // r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp +{ .mmi + st8 [r30]=r23,16 // rp + st8 [r31]=r18,16 // pr + mov r24=ar.pfs + ;; +} +{ .mmb + st8 [r30]=r24,16 // pfs + st8 [r31]=r20,16 // bspstore + cover + ;; +} +{ .mmi + mov r18=ar.fpsr + mov r23=cr.ipsr + extr.u r24=r20,61,3 + ;; +} + // r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr +{ .mmi + st8 [r30]=r19,16 // rnat + st8 [r31]=r0,16 // __spare + cmp.le p12,p13=5,r24 + ;; +} +{ .mmi + st8.spill [r30]=r13,16 // tp + st8 [r31]=r21,16 // rsc + tbit.nz p11,p10=r23,14 // p11=interrupts enabled + ;; +} +{ .mmi + st8 [r30]=r18,16 // fpsr +(p13) mov r20=ar.k6 // kernel register stack + nop 0 + ;; +} + // r20=bspstore, r22=iip, r23=ipsr +{ .mmi + st8 [r31]=r23,16 // psr + mov ar.bspstore=r20 + nop 0 + ;; +} +{ .mmi + mov r18=ar.bsp + ;; + mov r19=cr.ifs + sub r18=r18,r20 + ;; +} +{ .mmi + st8.spill [r30]=gp,16 // gp + st8 [r31]=r18,16 // ndirty + nop 0 + ;; +} + // r19=ifs, r22=iip +{ .mmi + st8 [r30]=r19,16 // cfm + st8 [r31]=r22,16 // iip + nop 0 + ;; +} +{ .mmi + st8 [r30]=r17 // ifa + mov r18=cr.isr + add r29=16,r30 + ;; +} +{ .mmi + st8 [r31]=r18 // isr + add r30=8,r29 + add r31=16,r29 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r2,16 // r2 + .mem.offset 8,0 + st8.spill [r31]=r3,16 // r3 + add r2=9*8,r29 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r8,16 // r8 + .mem.offset 8,0 + st8.spill [r31]=r9,16 // r9 + add r3=8,r2 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r10,16 // r10 + .mem.offset 8,0 + st8.spill [r31]=r11,16 // r11 + add r8=16,r16 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r14 // r14 + .mem.offset 8,0 + st8.spill [r31]=r15 // r15 + mov r9=r29 +} +{ .mmb + mov r10=ar.csd + mov r11=ar.ssd + bsw.1 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r16,16 // r16 + .mem.offset 8,0 + st8.spill [r3]=r17,16 // r17 + mov r14=b6 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r18,16 // r18 + .mem.offset 8,0 + st8.spill [r3]=r19,16 // r19 + mov r15=b7 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r20,16 // r20 + .mem.offset 8,0 + st8.spill [r3]=r21,16 // r21 + mov b7=r8 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r22,16 // r22 + .mem.offset 8,0 + st8.spill [r3]=r23,16 // r23 + ;; +} + + .mem.offset 0,0 + st8.spill [r2]=r24,16 // r24 + .mem.offset 8,0 + st8.spill [r3]=r25,16 // r25 + ;; + .mem.offset 0,0 + st8.spill [r2]=r26,16 // r26 + .mem.offset 8,0 + st8.spill [r3]=r27,16 // r27 + ;; + .mem.offset 0,0 + st8.spill [r2]=r28,16 // r28 + .mem.offset 8,0 + st8.spill [r3]=r29,16 // r29 + ;; + .mem.offset 0,0 + st8.spill [r2]=r30,16 // r30 + .mem.offset 8,0 + st8.spill [r3]=r31,16 // r31 + ;; + +{ .mmi + st8 [r2]=r14,16 // b6 + mov r17=ar.unat + nop 0 + ;; +} +{ .mmi + st8 [r3]=r15,16 // b7 + mov r16=ar.ccv + nop 0 + ;; +} +{ .mmi + st8 [r2]=r16,16 // ccv + st8 [r3]=r10,16 // csd + nop 0 + ;; +} +{ .mmi + st8 [r2]=r11,24 // ssd + st8 [r9]=r17 + nop 0 + ;; +} + + stf.spill [r3]=f6,32 // f6 + stf.spill [r2]=f7,32 // f7 + ;; + stf.spill [r3]=f8,32 // f8 + stf.spill [r2]=f9,32 // f9 + ;; + stf.spill [r3]=f10,32 // f10 + stf.spill [r2]=f11,32 // f11 + ;; + stf.spill [r3]=f12,32 // f12 + stf.spill [r2]=f13,32 // f13 + ;; + stf.spill [r3]=f14 // f14 + stf.spill [r2]=f15 // f15 + ;; +{ .mmi + mov ar.rsc=3 + mov r13=ar.k4 + nop 0 + ;; +} +{ .mlx + ssm psr.ic|psr.dfh + movl gp=__gp + ;; +} +{ .mfb + srlz.d + nop 0 + br.sptk b7 + ;; +} +END(exception_save) + +/* + * exception_restore: restore interrupted state + * + * Arguments: + * sp+16 trapframe pointer + */ +ENTRY(exception_restore, 0) +{ .mmi + rsm psr.i + add sp=16,sp + nop 0 + ;; +} +{ .mmi + add r3=SIZEOF_TRAPFRAME-32,sp + add r2=SIZEOF_TRAPFRAME-16,sp + add r8=SIZEOF_SPECIAL+16,sp + ;; +} + // The next load can trap. Let it be... + ldf.fill f15=[r2],-32 // f15 + ldf.fill f14=[r3],-32 // f14 + ;; + ldf.fill f13=[r2],-32 // f13 + ldf.fill f12=[r3],-32 // f12 + ;; + ldf.fill f11=[r2],-32 // f11 + ldf.fill f10=[r3],-32 // f10 + ;; + ldf.fill f9=[r2],-32 // f9 + ldf.fill f8=[r3],-32 // f8 + ;; + ldf.fill f7=[r2],-24 // f7 + ldf.fill f6=[r3],-16 // f6 + ;; + +{ .mmi + ld8 r8=[r8] // unat (after) + ;; + mov ar.unat=r8 + nop 0 + ;; +} + + ld8 r10=[r2],-16 // ssd + ld8 r11=[r3],-16 // csd + ;; + mov ar.ssd=r10 + mov ar.csd=r11 + + ld8 r14=[r2],-16 // ccv + ld8 r15=[r3],-16 // b7 + ;; + +{ .mmi + mov ar.ccv=r14 + ld8 r8=[r2],-16 // b6 + mov b7=r15 + ;; +} +{ .mmi + ld8.fill r31=[r3],-16 // r31 + ld8.fill r30=[r2],-16 // r30 + mov b6=r8 + ;; +} + + ld8.fill r29=[r3],-16 // r29 + ld8.fill r28=[r2],-16 // r28 + ;; + ld8.fill r27=[r3],-16 // r27 + ld8.fill r26=[r2],-16 // r26 + ;; + ld8.fill r25=[r3],-16 // r25 + ld8.fill r24=[r2],-16 // r24 + ;; + ld8.fill r23=[r3],-16 // r23 + ld8.fill r22=[r2],-16 // r22 + ;; + ld8.fill r21=[r3],-16 // r21 + ld8.fill r20=[r2],-16 // r20 + ;; + ld8.fill r19=[r3],-16 // r19 + ld8.fill r18=[r2],-16 // r18 + ;; + +{ .mmb + ld8.fill r17=[r3],-16 // r17 + ld8.fill r16=[r2],-16 // r16 + bsw.0 + ;; +} +{ .mmi + ld8.fill r15=[r3],-16 // r15 + ld8.fill r14=[r2],-16 // r14 + add r31=16,sp + ;; +} +{ .mmi + ld8 r16=[sp] // tf_length + ld8.fill r11=[r3],-16 // r11 + add r30=24,sp + ;; +} +{ .mmi + ld8.fill r10=[r2],-16 // r10 + ld8.fill r9=[r3],-16 // r9 + add r16=r16,sp // ar.k7 + ;; +} +{ .mmi + ld8.fill r8=[r2],-16 // r8 + ld8.fill r3=[r3] // r3 + ;; +} + // We want nested TLB faults from here on... + rsm psr.ic|psr.i + ld8.fill r2=[r2] // r2 + nop 0 + ;; + srlz.d + ld8.fill sp=[r31],16 // sp + nop 0 + ;; + + ld8 r17=[r30],16 // unat + ld8 r29=[r31],16 // rp + ;; + ld8 r18=[r30],16 // pr + ld8 r28=[r31],16 // pfs + mov rp=r29 + ;; + ld8 r20=[r30],24 // bspstore + ld8 r21=[r31],24 // rnat + mov ar.pfs=r28 + ;; + ld8.fill r29=[r30],16 // tp + ld8 r22=[r31],16 // rsc + ;; +{ .mmi + ld8 r23=[r30],16 // fpsr + ld8 r24=[r31],16 // psr + extr.u r28=r20,61,3 + ;; +} +{ .mmi + ld8.fill r1=[r30],16 // gp + ld8 r25=[r31],16 // ndirty + cmp.le p14,p15=5,r28 + ;; +} +{ .mmb + ld8 r26=[r30] // cfm + ld8 r19=[r31] // ip +(p14) br.cond.sptk 1f + ;; +} +{ .mib + // Switch register stack + alloc r30=ar.pfs,0,0,0,0 // discard current frame + shl r31=r25,16 // value for ar.rsc + nop 0 + ;; +} + // The loadrs can fault if the backing store is not currently + // mapped. We assured forward progress by getting everything we + // need from the trapframe so that we don't care if the CPU + // purges that translation when it needs to insert a new one for + // the backing store. +{ .mmi + mov ar.rsc=r31 // setup for loadrs + mov ar.k7=r16 + mov r13=r29 + ;; +} +exception_restore_restart: +{ .mmi + mov r30=ar.bspstore + ;; + loadrs // load user regs + nop 0 + ;; +} +{ .mmi + mov r31=ar.bspstore + ;; + mov ar.bspstore=r20 + dep r31=0,r31,0,9 + ;; +} +{ .mmb + mov ar.k6=r31 + mov ar.rnat=r21 + nop 0 + ;; +} +1: +{ .mmb + mov ar.unat=r17 + mov cr.iip=r19 + nop 0 +} +{ .mmi + mov cr.ipsr=r24 + mov cr.ifs=r26 + mov pr=r18,0x1fffe + ;; +} +{ .mmb + mov ar.rsc=r22 + mov ar.fpsr=r23 + rfi + ;; +} +END(exception_restore) + /* * Call exception_save_regs to preserve the interrupted state in a * trapframe. Note that we don't use a call instruction because we @@ -46,20 +576,27 @@ * which will restore the interrupted state before executing an rfi to * resume it. */ -#define TRAP(_n_) \ -1: mov r17=ip;; \ - add r17=2f-1b,r17; \ - mov r16=b6;; \ - mov b6=r17; \ - br.sptk.few exception_save; \ -2: (p3) ssm psr.i; \ - alloc r15=ar.pfs,0,0,3,0; \ - mov out0=_n_; \ - mov out1=r14; \ - mov out2=sp;; \ - add sp=-16,sp;; \ - br.call.sptk.few rp=trap; \ -3: br.sptk.many exception_restore +#define TRAP(_n_, _ifa_) \ +{ .mib ; \ + mov r17=_ifa_ ; \ + mov r16=ip ; \ + br.sptk exception_save ; \ +} ; \ +{ .mmi ; \ +(p11) ssm psr.i ;; \ + alloc r15=ar.pfs,0,0,2,0 ; \ + mov out0=_n_ ;; \ +} ; \ +{ .mfb ; \ + add out1=16,sp ; \ + nop 0 ; \ + br.call.sptk rp=trap ; \ +} ; \ +{ .mfb ; \ + nop 0 ; \ + nop 0 ; \ + br.sptk exception_restore ; \ +} #define IVT_ENTRY(name, offset) \ .org ia64_vector_table + offset; \ @@ -88,7 +625,7 @@ ivt_##name: ia64_vector_table: IVT_ENTRY(VHPT_Translation, 0x0000) - TRAP(0) + TRAP(0, cr.ifa) IVT_END(VHPT_Translation) IVT_ENTRY(Instruction_TLB, 0x0400) @@ -103,8 +640,8 @@ IVT_ENTRY(Instruction_TLB, 0x0400) ;; ld8 r21=[r21] // check VHPT tag ;; - cmp.ne p1,p0=r21,r19 -(p1) br.dpnt.few 1f + cmp.ne p15,p0=r21,r19 +(p15) br.dpnt.few 1f ;; ld8 r21=[r18] // read pte mov pr=r17,0x1ffff @@ -118,15 +655,15 @@ IVT_ENTRY(Instruction_TLB, 0x0400) ;; srlz.d // serialize ;; -2: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +2: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 3f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 3f // if not, read next in chain ;; ld8 r21=[r20],8 // read pte ;; @@ -165,7 +702,7 @@ IVT_ENTRY(Instruction_TLB, 0x0400) ;; srlz.d ;; - TRAP(20) // Page Not Present trap + TRAP(20, cr.ifa) // Page Not Present trap IVT_END(Instruction_TLB) IVT_ENTRY(Data_TLB, 0x0800) @@ -180,8 +717,8 @@ IVT_ENTRY(Data_TLB, 0x0800) ;; ld8 r21=[r21] // check VHPT tag ;; - cmp.ne p1,p0=r21,r19 -(p1) br.dpnt.few 1f + cmp.ne p15,p0=r21,r19 +(p15) br.dpnt.few 1f ;; ld8 r21=[r18] // read pte mov pr=r17,0x1ffff @@ -195,15 +732,15 @@ IVT_ENTRY(Data_TLB, 0x0800) ;; srlz.d // serialize ;; -2: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +2: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 3f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 3f // if not, read next in chain ;; ld8 r21=[r20],8 // read pte ;; @@ -242,7 +779,7 @@ IVT_ENTRY(Data_TLB, 0x0800) ;; srlz.d ;; - TRAP(20) // Page Not Present trap + TRAP(20, cr.ifa) // Page Not Present trap IVT_END(Data_TLB) IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) @@ -251,12 +788,12 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) ;; extr.u r17=r16,61,3 // get region number ;; - cmp.ge p3,p0=5,r17 // RR0-RR5? - cmp.eq p1,p2=7,r17 // RR7->p1, RR6->p2 -(p3) br.spnt 9f + cmp.ge p13,p0=5,r17 // RR0-RR5? + cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 +(p13) br.spnt 9f ;; -(p1) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX -(p2) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX +(p15) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX +(p14) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX ;; dep r16=0,r16,50,14 // clear bits above PPN ;; @@ -267,7 +804,7 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) ;; rfi 9: mov pr=r18,0x1ffff // restore predicates - TRAP(3) + TRAP(3, cr.ifa) IVT_END(Alternate_Instruction_TLB) IVT_ENTRY(Alternate_Data_TLB, 0x1000) @@ -276,12 +813,12 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000) ;; extr.u r17=r16,61,3 // get region number ;; - cmp.ge p3,p0=5,r17 // RR0-RR5? - cmp.eq p1,p2=7,r17 // RR7->p1, RR6->p2 -(p3) br.spnt 9f + cmp.ge p13,p0=5,r17 // RR0-RR5? + cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 +(p13) br.spnt 9f ;; -(p1) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW -(p2) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW +(p15) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW +(p14) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW ;; dep r16=0,r16,50,14 // clear bits above PPN ;; @@ -292,19 +829,92 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000) ;; rfi 9: mov pr=r18,0x1ffff // restore predicates - TRAP(4) + TRAP(4, cr.ifa) IVT_END(Alternate_Data_TLB) IVT_ENTRY(Data_Nested_TLB, 0x1400) - TRAP(5) + // See exception_save_restart and exception_restore_restart for the + // contexts that may cause a data nested TLB. We can only use the + // banked general registers and predicates, but don't use: + // p14 & p15 - Set in exception save + // r16 & r17 - Arguments to exception save + // r30 - Faulting address (modulo page size) + // We assume r30 has the virtual addresses that relate to the data + // nested TLB fault. The address does not have to be exact, as long + // as it's in the same page. We use physical addressing to avoid + // double nested faults. Since all virtual addresses we encounter + // here are direct mapped region 7 addresses, we have no problem + // constructing physical addresses. +{ .mlx + rsm psr.dt + movl r27=ia64_kptdir + ;; +} +{ .mii + srlz.d + dep r27=0,r27,61,3 + extr.u r28=r30,PAGE_SHIFT,61-PAGE_SHIFT + ;; +} +{ .mii + ld8 r27=[r27] + shr.u r29=r28,PAGE_SHIFT-5 // dir index + extr.u r28=r28,0,PAGE_SHIFT-5 // pte index + ;; +} +{ .mmi + shladd r27=r29,3,r27 + ;; + mov cr.ifa=r30 + dep r27=0,r27,61,3 + ;; +} +{ .mmi + ld8 r27=[r27] + mov r29=rr[r30] + shl r28=r28,5 + ;; +} +{ .mii + add r27=r27,r28 // address of pte + dep r29=0,r29,0,2 + ;; + dep r27=0,r27,61,3 + ;; +} +{ .mmi + ld8 r28=[r27] + ;; + mov cr.itir=r29 + or r28=PTE_D|PTE_A,r28 + ;; +} +{ .mlx + st8 [r27]=r28 + movl r29=exception_save_restart + ;; +} +{ .mmi + itc.d r28 + ;; + ssm psr.dt + cmp.eq p12,p13=r26,r29 + ;; +} +{ .mbb + srlz.d +(p12) br.sptk exception_save_restart +(p13) br.sptk exception_restore_restart + ;; +} IVT_END(Data_Nested_TLB) IVT_ENTRY(Instruction_Key_Miss, 0x1800) - TRAP(6) + TRAP(6, cr.ifa) IVT_END(Instruction_Key_Miss) IVT_ENTRY(Data_Key_Miss, 0x1c00) - TRAP(7) + TRAP(7, cr.ifa) IVT_END(Data_Key_Miss) IVT_ENTRY(Dirty_Bit, 0x2000) @@ -325,15 +935,15 @@ IVT_ENTRY(Dirty_Bit, 0x2000) ;; srlz.d // serialize ;; -1: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +1: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 2f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_D|PTE_A @@ -373,7 +983,7 @@ IVT_ENTRY(Dirty_Bit, 0x2000) br.cond.sptk.few 1b // loop 9: mov pr=r17,0x1ffff // restore predicates - TRAP(8) // die horribly + TRAP(8, cr.ifa) // die horribly IVT_END(Dirty_Bit) IVT_ENTRY(Instruction_Access_Bit, 0x2400) @@ -394,15 +1004,15 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400) ;; srlz.d // serialize ;; -1: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +1: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 2f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_A @@ -442,7 +1052,7 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400) br.cond.sptk.few 1b // loop 9: mov pr=r17,0x1ffff // restore predicates - TRAP(9) + TRAP(9, cr.ifa) IVT_END(Instruction_Access_Bit) IVT_ENTRY(Data_Access_Bit, 0x2800) @@ -463,15 +1073,15 @@ IVT_ENTRY(Data_Access_Bit, 0x2800) ;; srlz.d // serialize ;; -1: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +1: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 2f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_A @@ -511,1344 +1121,283 @@ IVT_ENTRY(Data_Access_Bit, 0x2800) br.cond.sptk.few 1b // loop 9: mov pr=r17,0x1ffff // restore predicates - TRAP(10) + TRAP(10, cr.ifa) IVT_END(Data_Access_Bit) IVT_ENTRY(Break_Instruction, 0x2c00) - mov r16=pr // save pr for a moment - mov r17=cr.iim;; // read break value - mov r18=0x100000;; // syscall number - cmp.ne p6,p0=r18,r17;; // check for syscall -(p6) br.dpnt.few 9f - - mov r17=cr.ipsr;; // check for user mode - extr.u r17=r17,32,2;; - cmp.eq p6,p0=r0,r17 -(p6) br.dpnt.few 9f // trap if kernel mode - - // Note: p6 and p7 are temporaries so we don't need to restore - // the value of pr here since the user-mode program assumes - // that syscalls only preserve the function-preserved state. - - br.sptk.many do_syscall +{ .mib + mov r17=cr.iim + mov r16=ip + br.sptk exception_save + ;; +} +{ .mmi + alloc r15=ar.pfs,0,0,2,0 + flushrs + mov out0=11 + ;; +} +{ .mib +(p11) ssm psr.i + add out1=16,sp + br.call.sptk rp=trap ;; -9: mov pr=r16,0x1ffff // restore pr - TRAP(11) +} +{ .mfb + nop 0 + nop 0 + br.sptk exception_restore + ;; +} IVT_END(Break_Instruction) IVT_ENTRY(External_Interrupt, 0x3000) - mov r16=b6 // save user's b6 -1: mov r17=ip;; // construct return address - add r17=2f-1b,r17;; // for exception_save - mov b6=r17 - br.sptk.few exception_save // 'call' exception_save - +{ .mib + mov r17=cr.lid // cr.iim and cr.ifa are undefined. + mov r16=ip + br.sptk exception_save + ;; +} +{ .mmb 2: alloc r14=ar.pfs,0,0,2,0 // make a frame for calling with - - mov out1=sp;; - add sp=-16,sp;; + add out1=16,sp + nop 0 +} 3: mov out0=cr.ivr // find interrupt vector ;; - cmp.eq p6,p0=15,out0 // check for spurious vector number -(p6) br.dpnt.few exception_restore // if spurious, we are done + cmp.eq p15,p0=15,out0 // check for spurious vector number +(p15) br.dpnt.few exception_restore // if spurious, we are done ;; ssm psr.i // re-enable interrupts - ;; // now that we are in-progress - srlz.d - ;; br.call.sptk.many rp=interrupt // call high-level handler - + ;; rsm psr.i // disable interrupts ;; srlz.d - ;; mov cr.eoi=r0 // and ack the interrupt ;; srlz.d br.sptk.few 3b // loop for more + ;; IVT_END(External_Interrupt) IVT_ENTRY(Reserved_3400, 0x3400) - TRAP(13) + TRAP(13, cr.ifa) IVT_END(Reserved_3400) IVT_ENTRY(Reserved_3800, 0x3800) - TRAP(14) + TRAP(14, cr.ifa) IVT_END(Reserved_3800) IVT_ENTRY(Reserved_3c00, 0x3c00) - TRAP(15) + TRAP(15, cr.ifa) IVT_END(Reserved_3c00) IVT_ENTRY(Reserved_4000, 0x4000) - TRAP(16) + TRAP(16, cr.ifa) IVT_END(Reserved_4000) IVT_ENTRY(Reserved_4400, 0x4400) - TRAP(17) + TRAP(17, cr.ifa) IVT_END(Reserved_4400) IVT_ENTRY(Reserved_4800, 0x4800) - TRAP(18) + TRAP(18, cr.ifa) IVT_END(Reserved_4800) IVT_ENTRY(Reserved_4c00, 0x4c00) - TRAP(19) + TRAP(19, cr.ifa) IVT_END(Reserved_4c00) IVT_ENTRY(Page_Not_Present, 0x5000) - TRAP(20) + TRAP(20, cr.ifa) IVT_END(Page_Not_Present) IVT_ENTRY(Key_Permission, 0x5100) - TRAP(21) + TRAP(21, cr.ifa) IVT_END(Key_Permission) IVT_ENTRY(Instruction_Access_Rights, 0x5200) - TRAP(22) + TRAP(22, cr.ifa) IVT_END(Instruction_Access_Rights) IVT_ENTRY(Data_Access_Rights, 0x5300) - TRAP(23) + TRAP(23, cr.ifa) IVT_END(Data_Access_Rights) IVT_ENTRY(General_Exception, 0x5400) - TRAP(24) + TRAP(24, cr.ifa) IVT_END(General_Exception) IVT_ENTRY(Disabled_FP_Register, 0x5500) - TRAP(25) + TRAP(25, cr.ifa) IVT_END(Disabled_FP_Register) IVT_ENTRY(NaT_Consumption, 0x5600) - TRAP(26) + TRAP(26, cr.ifa) IVT_END(NaT_Consumption) IVT_ENTRY(Speculation, 0x5700) - TRAP(27) + TRAP(27, cr.iim) IVT_END(Speculation) IVT_ENTRY(Reserved_5800, 0x5800) - TRAP(28) + TRAP(28, cr.ifa) IVT_END(Reserved_5800) IVT_ENTRY(Debug, 0x5900) - TRAP(29) + TRAP(29, cr.ifa) IVT_END(Debug) IVT_ENTRY(Unaligned_Reference, 0x5a00) - TRAP(30) + TRAP(30, cr.ifa) IVT_END(Unaligned_Reference) IVT_ENTRY(Unsupported_Data_Reference, 0x5b00) - TRAP(31) + TRAP(31, cr.ifa) IVT_END(Unsupported_Data_Reference) IVT_ENTRY(Floating_Point_Fault, 0x5c00) - TRAP(32) + TRAP(32, cr.ifa) IVT_END(Floating_Point_Fault) IVT_ENTRY(Floating_Point_Trap, 0x5d00) - TRAP(33) + TRAP(33, cr.ifa) IVT_END(Floating_Point_Trap) IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00) - TRAP(34) + TRAP(34, cr.ifa) IVT_END(Lower_Privilege_Transfer_Trap) IVT_ENTRY(Taken_Branch_Trap, 0x5f00) - TRAP(35) + TRAP(35, cr.ifa) IVT_END(Taken_Branch_Trap) IVT_ENTRY(Single_Step_Trap, 0x6000) - TRAP(36) + TRAP(36, cr.ifa) IVT_END(Single_Step_Trap) IVT_ENTRY(Reserved_6100, 0x6100) - TRAP(37) + TRAP(37, cr.ifa) IVT_END(Reserved_6100) IVT_ENTRY(Reserved_6200, 0x6200) - TRAP(38) + TRAP(38, cr.ifa) IVT_END(Reserved_6200) IVT_ENTRY(Reserved_6300, 0x6300) - TRAP(39) + TRAP(39, cr.ifa) IVT_END(Reserved_6300) IVT_ENTRY(Reserved_6400, 0x6400) - TRAP(40) + TRAP(40, cr.ifa) IVT_END(Reserved_6400) IVT_ENTRY(Reserved_6500, 0x6500) - TRAP(41) + TRAP(41, cr.ifa) IVT_END(Reserved_6500) IVT_ENTRY(Reserved_6600, 0x6600) - TRAP(42) + TRAP(42, cr.ifa) IVT_END(Reserved_6600) IVT_ENTRY(Reserved_6700, 0x6700) - TRAP(43) + TRAP(43, cr.ifa) IVT_END(Reserved_6700) IVT_ENTRY(Reserved_6800, 0x6800) - TRAP(44) + TRAP(44, cr.ifa) IVT_END(Reserved_6800) IVT_ENTRY(IA_32_Exception, 0x6900) - TRAP(45) + TRAP(45, cr.ifa) IVT_END(IA_32_Exception) IVT_ENTRY(IA_32_Intercept, 0x6a00) - TRAP(46) + TRAP(46, cr.iim) IVT_END(IA_32_Intercept) IVT_ENTRY(IA_32_Interrupt, 0x6b00) - TRAP(47) + TRAP(47, cr.ifa) IVT_END(IA_32_Interrupt) IVT_ENTRY(Reserved_6c00, 0x6c00) - TRAP(48) + TRAP(48, cr.ifa) IVT_END(Reserved_6c00) IVT_ENTRY(Reserved_6d00, 0x6d00) - TRAP(49) + TRAP(49, cr.ifa) IVT_END(Reserved_6d00) IVT_ENTRY(Reserved_6e00, 0x6e00) - TRAP(50) + TRAP(50, cr.ifa) IVT_END(Reserved_6e00) IVT_ENTRY(Reserved_6f00, 0x6f00) - TRAP(51) + TRAP(51, cr.ifa) IVT_END(Reserved_6f00) IVT_ENTRY(Reserved_7000, 0x7000) - TRAP(52) + TRAP(52, cr.ifa) IVT_END(Reserved_7000) IVT_ENTRY(Reserved_7100, 0x7100) - TRAP(53) + TRAP(53, cr.ifa) IVT_END(Reserved_7100) IVT_ENTRY(Reserved_7200, 0x7200) - TRAP(54) + TRAP(54, cr.ifa) IVT_END(Reserved_7200) IVT_ENTRY(Reserved_7300, 0x7300) - TRAP(55) + TRAP(55, cr.ifa) IVT_END(Reserved_7300) IVT_ENTRY(Reserved_7400, 0x7400) - TRAP(56) + TRAP(56, cr.ifa) IVT_END(Reserved_7400) IVT_ENTRY(Reserved_7500, 0x7500) - TRAP(57) + TRAP(57, cr.ifa) IVT_END(Reserved_7500) IVT_ENTRY(Reserved_7600, 0x7600) - TRAP(58) + TRAP(58, cr.ifa) IVT_END(Reserved_7600) IVT_ENTRY(Reserved_7700, 0x7700) - TRAP(59) + TRAP(59, cr.ifa) IVT_END(Reserved_7700) IVT_ENTRY(Reserved_7800, 0x7800) - TRAP(60) + TRAP(60, cr.ifa) IVT_END(Reserved_7800) IVT_ENTRY(Reserved_7900, 0x7900) - TRAP(61) + TRAP(61, cr.ifa) IVT_END(Reserved_7900) IVT_ENTRY(Reserved_7a00, 0x7a00) - TRAP(62) + TRAP(62, cr.ifa) IVT_END(Reserved_7a00) IVT_ENTRY(Reserved_7b00, 0x7b00) - TRAP(63) + TRAP(63, cr.ifa) IVT_END(Reserved_7b00) IVT_ENTRY(Reserved_7c00, 0x7c00) - TRAP(64) + TRAP(64, cr.ifa) IVT_END(Reserved_7c00) IVT_ENTRY(Reserved_7d00, 0x7d00) - TRAP(65) + TRAP(65, cr.ifa) IVT_END(Reserved_7d00) IVT_ENTRY(Reserved_7e00, 0x7e00) - TRAP(66) + TRAP(66, cr.ifa) IVT_END(Reserved_7e00) IVT_ENTRY(Reserved_7f00, 0x7f00) - TRAP(67) + TRAP(67, cr.ifa) IVT_END(Reserved_7f00) - - .text - -/* - * exception_restore: restore interrupted state - * - * Arguments: - * sp+16 trapframe pointer - * r4 ar.pfs before the alloc in TRAP() - * - */ -ENTRY(exception_restore, 0) -{ .mfi - alloc r14=ar.pfs,0,0,1,0 // in case we call ast() - nop 1 - add r3=TF_CR_IPSR+16,sp - ;; -} -{ .mmi - ld8 r30=[r3] // ipsr - ;; - nop 2 - extr.u r16=r30,32,2 // extract ipsr.cpl - ;; -} -{ .mfb - cmp.eq p1,p2=r0,r16 // test for return to kernel mode - nop 3 -(p1) br.cond.dpnt 2f // no ast check for returns to kernel -} -3: -{ .mmi - add r3=PC_CURTHREAD,r13 // &curthread - ;; - ld8 r3=[r3] // curthread - add r2=(TDF_ASTPENDING|TDF_NEEDRESCHED),r0 - ;; -} -{ .mmb - mov r15=psr // save interrupt enable status - nop 4 - ;; -} -{ .mmi - ;; - rsm psr.i // disable interrupts - add r3=TD_FLAGS,r3 // &curthread->td_flags - ;; -} -{ .mmi - ld4 r14=[r3] // fetch curthread->td_flags - ;; - and r14=r2,r14 // flags & (TDF_ASTPENDING|TDF_NEEDRESCHED) - nop 5 - ;; -} -{ .mfb - cmp4.eq p6,p7=r0,r14 // == 0 ? - nop 6 -(p6) br.cond.dptk 2f - ;; -} -{ .mmi - mov psr.l=r15 // restore interrups - ;; - srlz.d - add out0=16,sp // trapframe argument to ast() -} -{ .bbb - br.call.sptk.many rp=ast // note: p1, p2 preserved - br.sptk 3b - nop 7 -} -2: -{ .mmi - rsm psr.ic|psr.dt|psr.i // disable interrupt collection and vm - ;; - srlz.i - add r3=16,sp - ;; -} -{ .mmi -(p2) add r16=SIZEOF_TRAPFRAME+16,sp // restore ar.k6 (kernel sp) - ;; -(p2) mov ar.k6=r16 - dep r3=0,r3,61,3 // physical address - ;; -} -{ .mmi - add r1=SIZEOF_TRAPFRAME-16,r3 // r1=&tf_f[FRAME_F15] - ;; - ldf.fill f15=[r1],-32 // r1=&tf_f[FRAME_F13] - add r2=SIZEOF_TRAPFRAME-32,r3 // r2=&tf_f[FRAME_F14] - ;; -} -{ .mmb - ldf.fill f14=[r2],-32 // r2=&tf_f[FRAME_F12] - ldf.fill f13=[r1],-32 // r1=&tf_f[FRAME_F11] - nop 8 - ;; -} -{ .mmi - ldf.fill f12=[r2],-32 // r2=&tf_f[FRAME_F10] - ldf.fill f11=[r1],-32 // r1=&tf_f[FRAME_F9] - nop 9 - ;; -} -{ .mmb - ldf.fill f10=[r2],-32 // r2=&tf_f[FRAME_F8] - ldf.fill f9=[r1],-32 // r1=&tf_f[FRAME_F7] - nop 10 - ;; -} -{ .mmi - ldf.fill f8=[r2],-32 // r2=&tf_f[FRAME_F6] - ldf.fill f7=[r1],-24 // r1=&tf_r[FRAME_R31] - nop 11 - ;; -} -{ .mmb - ldf.fill f6=[r2],-16 // r2=&tf_r[FRAME_R30] - ld8.fill r31=[r1],-16 // r1=&tf_r[FRAME_R29] - nop 12 - ;; -} -{ .mmi - ld8.fill r30=[r2],-16 // r2=&tf_r[FRAME_R28] - ld8.fill r29=[r1],-16 // r1=&tf_r[FRAME_R27] - nop 13 - ;; -} -{ .mmb - ld8.fill r28=[r2],-16 // r2=&tf_r[FRAME_R26] - ld8.fill r27=[r1],-16 // r1=&tf_r[FRAME_R25] - nop 14 - ;; -} -{ .mmi - ld8.fill r26=[r2],-16 // r2=&tf_r[FRAME_R24] - ld8.fill r25=[r1],-16 // r1=&tf_r[FRAME_R23] - nop 15 - ;; -} -{ .mmb - ld8.fill r24=[r2],-16 // r2=&tf_r[FRAME_R22] - ld8.fill r23=[r1],-16 // r1=&tf_r[FRAME_R21] - nop 16 - ;; -} -{ .mmi - ld8.fill r22=[r2],-16 // r2=&tf_r[FRAME_R20] - ld8.fill r21=[r1],-16 // r1=&tf_r[FRAME_R19] - nop 17 - ;; -} -{ .mmb - ld8.fill r20=[r2],-16 // r2=&tf_r[FRAME_R18] - ld8.fill r19=[r1],-16 // r1=&tf_r[FRAME_R17] - nop 18 - ;; -} -{ .mmi - ld8.fill r18=[r2],-16 // r2=&tf_r[FRAME_R16] - ld8.fill r17=[r1],-16 // r1=&tf_r[FRAME_R15] - nop 19 - ;; -} -{ .mfb - ld8.fill r16=[r2],-16 // r2=&tf_r[FRAME_R14] - nop 20 - bsw.0 // switch to bank 0 - ;; -} -{ .mmi - ld8.fill r15=[r1],-16 // r1=&tf_r[FRAME_R13] - ld8.fill r14=[r2],-16 // r2=&tf_r[FRAME_R12] - nop 21 - ;; -} - // Don't restore r13 if returning to kernel -{ .mmi - .pred.rel.mutex p1,p2 -(p2) ld8.fill r13=[r1],-16 // r1=&tf_r[FRAME_R11] - ld8.fill r12=[r2],-16 // r2=&tf_r[FRAME_R10] -(p1) add r1=-16,r1 // r1=&tf_r[FRAME_R11] - ;; -} -{ .mmb - ld8.fill r11=[r1],-16 // r1=&tf_r[FRAME_R9] - ld8.fill r10=[r2],-16 // r2=&tf_r[FRAME_R8] - nop 22 - ;; -} -{ .mmi - ld8.fill r9=[r1],-16 // r1=&tf_r[FRAME_R7] - ld8.fill r8=[r2],-16 // r2=&tf_r[FRAME_R6] - nop 23 - ;; -} -{ .mmb - ld8.fill r7=[r1],-16 // r1=&tf_r[FRAME_R5] - ld8.fill r6=[r2],-16 // r2=&tf_r[FRAME_R4] - nop 24 - ;; -} -{ .mmi - ld8.fill r5=[r1],-16 // r1=&tf_r[FRAME_R3] - ld8.fill r4=[r2],-16 // r2=&tf_r[FRAME_R2] - nop 25 - ;; -} -{ .mmb - ld8.fill r3=[r1],-16 // r1=&tf_r[FRAME_R1] - ld8.fill r23=[r2],-16 // r2=&tf_b[7] - nop 26 - ;; -} -{ .mmi - ld8.fill r24=[r1],-16 // r1=&tf_b[6] - ld8 r16=[r2],-16 // r16=b7, r2=&tf_b[5] - nop 27 - ;; -} -{ .mmi - ld8 r17=[r1],-16 // r17=b6, r1=&tf_b[4] - ld8 r18=[r2],-16 // r18=b5, r2=&tf_b[3] - mov b7=r16 - ;; -} -{ .mmi - ld8 r16=[r1],-16 // r16=b4, r1=&tf_b[2] - ld8 r19=[r2],-16 // r19=b3, r2=&tf_b[1] - mov b6=r17 - ;; -} -{ .mii - ld8 r17=[r1],-16 // r17=b2, r1=&tf_b[0] - mov b5=r18 - mov b4=r16 - ;; -} -{ .mii - ld8 r16=[r2],-16 // r16=b1, r2=&tf_ar_ec - mov b3=r19 - mov b2=r17 - ;; -} -{ .mmi - ld8 r17=[r1],-16 // r17=b0, r1=&tf_ar_lc - ld8 r18=[r2],-16 // r18=ar.ec, r2=&tf_ar_fptr - mov b1=r16 - ;; -} -{ .mmi - ld8 r16=[r1],-16 // r16=ar.lc, r1=&tf_ar_ccv - ld8 r19=[r2],-16 // r19=ar.fpsr, r1=&tf_ar_unat - mov b0=r17 - ;; -} -{ .mmi - ld8 r17=[r1],-16 // r17=ar.ccv, r1=&tf_ndirty - mov ar.fpsr=r19 - mov ar.ec=r18 - ;; -} -{ .mmi - ld8 r18=[r2],-16 // r18=ar.unat, r2=&tf_ar_rnat - mov ar.ccv=r17 - mov ar.lc=r16 - ;; -} -{ .mmb - ld8 r27=[r1],-16 // r1=&tf_ar_bspstore - ld8 r21=[r2],-16 // r2=&tf_cr_ifs - nop 28 - ;; -} -{ .mmi - mov ar.unat=r18 - ld8 r22=[r1],-16 // r1=&tf_ar_pfs - nop 29 -} -{ .mfb - ld8 r25=[r2],-16 // r2=&tf_ar_rsc - nop 30 -(p1) br.cond.dpnt.few 1f // don't switch bs if kernel - ;; -} -{ .mmi - alloc r16=ar.pfs,0,0,0,0 // discard current frame - ;; - nop 31 - shl r16=r27,16 // value for ar.rsc - ;; -} -{ .mmi - mov ar.rsc=r16 // setup for loadrs - ;; - loadrs // load user regs from kernel bs - nop 32 - ;; -} -{ .mmi - mov ar.bspstore=r22 - ;; - mov ar.rnat=r21 - nop 33 - ;; -} -1: -{ .mmb - ld8 r19=[r1],-16 // r1=&tf_pr - ld8 r20=[r2],-16 // r2=&tf_cr_ifa - nop 34 - ;; -} -{ .mmi - ld8 r27=[r1],-16 // r1=&tf_cr_isr - ld8 r28=[r2],-16 // r2=&tf_cr_ipsr - mov ar.pfs=r19 - ;; -} -{ .mmi - ld8 r29=[r1],-16 // r1=&tf_cr_iip - ld8 r30=[r2] - mov pr=r27,0x1ffff - ;; -} -{ .mmi - ld8 r31=[r1] - mov cr.ifs=r25 - mov r2=r23 - ;; -} -{ .mmi - mov cr.ifa=r28 - mov cr.iip=r31 - mov r1=r24 - ;; -} -{ .mmi - mov cr.ipsr=r30 - mov ar.rsc=r20 - nop 35 - ;; -} -{ .bbb - nop 36 - nop 37 - rfi - ;; -} -END(exception_restore) - - -/* - * exception_save: save interrupted state - * - * Arguments: - * b6 return address - * r16 saved b6 - * - * Return: - * r14 cr.iim value for break traps - * sp kernel stack pointer - * p1 true if user mode - * p2 true if kernel mode - * p3 true if interrupts were enabled - */ -ENTRY(exception_save, 0) -{ .mmi - rsm psr.dt // turn off data translations - ;; - srlz.d // serialize - mov r27=pr -} -{ .mmi - mov r30=cr.ipsr - ;; - mov r31=cr.iip - tbit.nz p3,p0=r30,14 // check for interrupt enable state -} -{ .mmi - mov r29=cr.isr - ;; - mov r26=sp // save sp - extr.u r17=r30,32,2 // extract ipsr.cpl - ;; -} -{ .mmi - cmp.eq p1,p2=r0,r17 // test for kernel mode - ;; -(p2) mov sp=ar.k6 // and switch to kernel stack - mov r24=r1 - ;; -} -{ .mii - mov r28=cr.ifa - add sp=-SIZEOF_TRAPFRAME,sp // reserve trapframe - ;; - dep r1=0,sp,61,3 // r1=&tf_flags - ;; -} -{ .mmi - st8 [r1]=r0,8 // zero flags, r1=&tf_cr_iip - ;; - mov r23=r2 - add r2=8,r1 // r2=&tf_cr_ipsr - ;; -} -{ .mmb - st8 [r1]=r31,16 // r1=&tf_cr_isr - st8 [r2]=r30,16 // r2=&tf_cr_ifa - nop 1 - ;; -} -{ .mmb - st8 [r1]=r29,16 // r1=&tf_pr - st8 [r2]=r28,16 // r2=&tf_ar_rsc - nop 2 - ;; -} -{ .mmi - st8 [r1]=r27,16 // r1=&tf_cr_pfs - mov r20=ar.rsc - mov r19=ar.pfs - ;; -} -{ .mmb - st8 [r2]=r20,16 // r2=&tf_cr_ifs - st8 [r1]=r19,16 // r1=&tf_ar_bspstore - cover - ;; -} -{ .mmi - mov ar.rsc=0 - ;; - mov r22=ar.bspstore - ;; -} -{ .mmi - mov r25=cr.ifs - mov r21=ar.rnat -(p1) mov r31=r22 // so we can figure out ndirty - ;; -} -{ .mmb -(p2) mov r31=ar.k5 // kernel backing store - st8 [r2]=r25,16 // r2=&tf_ar_rnat - nop 3 - ;; -} -{ .mmi - st8 [r1]=r22,16 // r1=&tf_ndirty -(p2) mov ar.bspstore=r31 // switch bspstore - nop 4 - ;; -} -{ .mmi - mov r17=ar.bsp - ;; - st8 [r2]=r21,16 // r2=&tf_ar_unat - sub r17=r17,r31 // ndirty (in bytes) - ;; -} -{ .mmi - st8 [r1]=r17,16 // r1=&tf_ar_ccv - mov ar.rsc=3 // switch RSE back on - mov r18=ar.lc -} -{ .mmi - mov r31=ar.unat - ;; - st8 [r2]=r31,16 // r2=&tf_ar_fpsr - mov r19=ar.ec -} -{ .mmi - mov r17=ar.ccv - ;; - st8 [r1]=r17,16 // r1=&tf_ar_lc - nop 5 - ;; -} -{ .mmi - mov r31=ar.fpsr - ;; - st8 [r2]=r31,16 // r2=&tf_ar_ec - mov r30=b0 - ;; -} -{ .mmi - st8 [r1]=r18,16 // r1=&tf_b[0] - ;; - st8 [r2]=r19,16 // r2=&tf_b[1] - mov r31=b1 -} -{ .mmi - st8 [r1]=r30,16 // r1=&tf_b[2] - ;; - st8 [r2]=r31,16 // r2=&tf_b[3] - mov r31=b2 - ;; -} -{ .mii - st8 [r1]=r31,16 // r1=&tf_b[4] - mov r17=b3 - ;; - mov r18=b4 -} -{ .mmi - st8 [r2]=r17,16 // r2=&tf_b[5] - ;; - st8 [r1]=r18,16 // r1=&tf_b[6] - mov r31=b5 - ;; -} -{ .mii - st8 [r2]=r31,16 // r2=&tf_b[7] - mov r18=b7 -} -{ .mmi - st8 [r1]=r16,16 // r1=&tf_r[FRAME_R1] - ;; - st8 [r2]=r18,16 // r2=&tf_r[FRAME_R2] - nop 7 - ;; -} -{ .mmb - .mem.offset 0,0 - st8.spill [r1]=r24,16 // r1=&tf_r[FRAME_R3] - .mem.offset 8,0 - st8.spill [r2]=r23,16 // r2=&tf_r[FRAME_R4] - nop 8 - ;; -} -{ .mmi - .mem.offset 16,0 - st8.spill [r1]=r3,16 // r1=&tf_r[FRAME_R5] - .mem.offset 32,0 - st8.spill [r2]=r4,16 // r2=&tf_r[FRAME_R6] - nop 9 - ;; -} -{ .mmb - .mem.offset 48,0 - st8.spill [r1]=r5,16 // r1=&tf_r[FRAME_R7] - .mem.offset 64,0 - st8.spill [r2]=r6,16 // r2=&tf_r[FRAME_R8] - nop 10 - ;; -} -{ .mmi - .mem.offset 80,0 - st8.spill [r1]=r7,16 // r1=&tf_r[FRAME_R9] - .mem.offset 96,0 - st8.spill [r2]=r8,16 // r2=&tf_r[FRAME_R10] - nop 11 - ;; -} -{ .mmb - .mem.offset 112,0 - st8.spill [r1]=r9,16 // r1=&tf_r[FRAME_R11] - .mem.offset 128,0 - st8.spill [r2]=r10,16 // r2=&tf_r[FRAME_SP] - nop 12 - ;; -} -{ .mmi - .mem.offset 144,0 - st8.spill [r1]=r11,16 // r1=&tf_r[FRAME_R13] - .mem.offset 160,0 - st8.spill [r2]=r26,16 // r2=&tf_r[FRAME_R14] - nop 13 - ;; -} -{ .mmb - .mem.offset 176,0 - st8.spill [r1]=r13,16 // r1=&tf_r[FRAME_R15] - .mem.offset 192,0 - st8.spill [r2]=r14,16 // r2=&tf_r[FRAME_R16] - nop 14 - ;; -} -{ .mfb - .mem.offset 208,0 - st8.spill [r1]=r15,16 // r1=&tf_r[FRAME_R17] - nop 15 - bsw.1 // switch to bank 1 - ;; -} -{ .mmi - .mem.offset 224,0 - st8.spill [r2]=r16,16 // r2=&tf_r[FRAME_R18] - .mem.offset 240,0 - st8.spill [r1]=r17,16 // r1=&tf_r[FRAME_R19] - nop 16 - ;; -} -{ .mmb - .mem.offset 256,0 - st8.spill [r2]=r18,16 // r2=&tf_r[FRAME_R20] - .mem.offset 272,0 - st8.spill [r1]=r19,16 // r1=&tf_r[FRAME_R21] - nop 17 - ;; -} -{ .mmi - .mem.offset 288,0 - st8.spill [r2]=r20,16 // r2=&tf_r[FRAME_R22] - .mem.offset 304,0 - st8.spill [r1]=r21,16 // r1=&tf_r[FRAME_R23] - nop 18 - ;; -} -{ .mmb - .mem.offset 320,0 - st8.spill [r2]=r22,16 // r2=&tf_r[FRAME_R24] - .mem.offset 336,0 - st8.spill [r1]=r23,16 // r1=&tf_r[FRAME_R25] - nop 19 - ;; -} -{ .mmi - .mem.offset 352,0 - st8.spill [r2]=r24,16 // r2=&tf_r[FRAME_R26] - .mem.offset 368,0 - st8.spill [r1]=r25,16 // r1=&tf_r[FRAME_R27] - nop 20 - ;; -} -{ .mmb - .mem.offset 384,0 - st8.spill [r2]=r26,16 // r2=&tf_r[FRAME_R28] - .mem.offset 400,0 - st8.spill [r1]=r27,16 // r1=&tf_r[FRAME_R29] - nop 21 - ;; -} -{ .mmi - .mem.offset 416,0 - st8.spill [r2]=r28,16 // r2=&tf_r[FRAME_R30] - .mem.offset 432,0 - st8.spill [r1]=r29,16 // r1=&tf_r[FRAME_R31] - nop 22 - ;; -} -{ .mmb - .mem.offset 448,0 - st8.spill [r2]=r30,16 // r2=&tf_f[FRAME_F6] - .mem.offset 464,0 - st8.spill [r1]=r31,24 // r1=&tf_f[FRAME_F7] - nop 23 - ;; -} -{ .mmi - stf.spill [r2]=f6,32 // r2=&tf_f[FRAME_F8] - stf.spill [r1]=f7,32 // r1=&tf_f[FRAME_F9] - nop 24 - ;; -} -{ .mmb - stf.spill [r2]=f8,32 // r2=&tf_f[FRAME_F10] - stf.spill [r1]=f9,32 // r1=&tf_f[FRAME_F11] - nop 25 - ;; -} -{ .mmi - stf.spill [r2]=f10,32 // r2=&tf_f[FRAME_F12] - stf.spill [r1]=f11,32 // r1=&tf_f[FRAME_F13] - nop 26 - ;; -} -{ .mmb - stf.spill [r2]=f12,32 // r2=&tf_f[FRAME_F14] - stf.spill [r1]=f13,32 // r1=&tf_f[FRAME_F15] - nop 27 - ;; -} -{ .mmi - stf.spill [r2]=f14 // - stf.spill [r1]=f15 // - nop 28 - ;; -} -{ .mlx - mov r14=cr.iim // break immediate - movl r1=__gp // kernel globals -} -{ .mmi - ssm psr.ic|psr.dt // enable interrupts & translation - ;; - srlz.i // serialize - nop 29 - ;; -} -{ .mfb - mov r13=ar.k4 // processor globals - nop 30 - br.sptk.few b6 // not br.ret - we were not br.call'ed - ;; -} -END(exception_save) - -/* - * System call entry point (via Break Instruction vector). - * - * Arguments: - * r15 System call number - * out0-out7 System call arguments - */ -ENTRY(do_syscall, 0) - .prologue - .unwabi @svr4, 'I' - .save rp,r0 - .body - // Save minimal state for syscall. - // We need to save enough state so that sendsig doesn't - // trash things if we take a signal during the system call. - // Essentially we need to save all the function-preserved - // state. Note that if we don't take a signal, we don't need - // to restore much of that state on the way out. Note also - // that when we save r4-r7 we spill their NaT bits into - // ar.unat. This register is preserved by the call to - // syscall() and if a full restore is needed, - // exception_restore will recover the NaT bits from ar.unat. - // The function-preserved state (including syscall number) is: - // - // r1,r4-r7,sp,r15 - // f16-f31 - // p1-p5,p16-p63 - // b0-b5 - // various ar's - // -{ .mmi // start reading high latency regs - mov r16=cr.ipsr // (13) - mov.m r17=ar.rsc // (13) - mov r18=sp // save user sp - ;; -} { .mmi - mov sp=ar.k6 // (13) kernel sp - mov r19=cr.isr // (13) - nop.i 0 -} { .mmi - mov.m ar.rsc=0 - ;; - mov.m r20=ar.bspstore // (13) - nop.i 0 -} { .mmi - mov.m r21=ar.k5 // (13) - mov.m r22=ar.rnat // (6) - nop.i 0 -} { .mmi - mov.m r23=ar.unat // (6) - rsm psr.dt // (5) physical addressing -} { .mii - mov r24=cr.iip // (2) - mov.i r25=ar.pfs // (2) - add sp=-SIZEOF_TRAPFRAME,sp // reserve trapframe - ;; -} { .mii - addl r27=FRAME_SYSCALL,r0 // (1) - mov r26=pr // (2) - dep r30=0,sp,61,3 // physical address -} { .mmi - srlz.d // serialize psr.dt - ;; - add r31=8,r30 // secondary pointer - ;; -} { .mmi - st8 [r30]=r27,16 // tf_flags - st8 [r31]=r24,16 // save cr.iip - mov r28=b0 - ;; -} { .mmi - st8 [r30]=r16,24 // save cr.ipsr, skip to pr - st8 [r31]=r19,24 // save cr.isr, skip to ar.rsc - mov r24=b1 - ;; -} { .mmi - st8 [r30]=r26,16 // save pr, skip to ar.pfs - st8 [r31]=r17,24 // save ar.rsc, skip to ar.bspstore - mov r27=b2 - ;; -} { .mii - st8 [r30]=r25,24 // save ar.pfs, skip to ar.rnat - mov r16=b3 - mov r17=b4 - ;; -} { .mmi - st8 [r31]=r20,24 // save ar.bspstore, skip to ar.unat - mov.m ar.bspstore=r21 // switch to kernel backing store - mov r29=b5 - ;; -} { .mmi - mov.m r20=ar.ccv - mov.m r21=ar.fpsr - nop.i 0 - ;; -} { .mmi - st8 [r30]=r22,24 // save ar.rnat, skip to ar.ccv - st8 [r31]=r23,16 // save ar.unat, skip to ar.fpsr - nop.i 0 - ;; -} { .mmi - st8 [r30]=r20,32 // save ar.ccv, skip to b0 - st8 [r31]=r21,32 // save ar.fpsr, skip to b1 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r28,16 // save b0, skip to b2 - st8 [r31]=r24,16 // save b1, skip to b3 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r27,16 // save b2, skip to b4 - st8 [r31]=r16,16 // save b3, skip to b5 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r17,TF_R_R1-(TF_B+4*8) // save b4, skip to r1 - st8 [r31]=r29,TF_R_R4-(TF_B+5*8) // save b5, skip to r4 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r1,TF_R_R5-TF_R_R1 // save r1, skip to r5 - .mem.offset 8,0 - st8.spill [r31]=r4,16 // save r4, skip to r6 - nop.i 0 - ;; -} { .mmi - .mem.offset 0,0 - st8.spill [r30]=r5,16 // save r5, skip to r7 - .mem.offset 8,0 - st8.spill [r31]=r6,TF_R_SP-TF_R_R6 // save r6, skip to sp - nop.i 0 - ;; -} { .mmi - .mem.offset 0,0 - st8.spill [r30]=r7,TF_R_R15-TF_R_R7 // save r7, skip to r15 - st8 [r31]=r18 // save sp - nop.i 0 - ;; -} { .mmb - st8 [r30]=r15 // save r15 (syscall number) - add sp=-(8*8),sp // reserve stack for arguments - br.call.sptk.few b0=Lsaveargs // dump args -} { .mmb - mov.m r13=ar.k4 // processor globals - nop.m 0 - bsw.1 // switch back to bank 1 - ;; -} { .mmb - mov r16=sp // point at args - mov.m r17=ar.k5 // for calculating ndirty - cover // preserve user register frame - ;; -} { .mmi - mov r18=cr.ifs // record user's CFM - mov.m r19=ar.bsp // ndirty = ar.bsp - kbsp - add sp=-16,sp // reserve scratch space - ;; -} { .mmi - add r20=TF_CR_IFS+(8*8),r16 // point at cr.ifs - ssm psr.ic|psr.dt // reenable traps and translation - sub r19=r19,r17 // calculate ndirty - ;; -} { .mmi - srlz.i // serialize psr.ic and psr.dt - ;; - ssm psr.i // safe to take interrupts again - add r21=TF_NDIRTY+(8*8),r16 // point at ndirty - ;; -} { .mmi - st8 [r20]=r18 // save cr.ifs - st8 [r21]=r19 // save ndirty - ;; -} { .mmi - alloc r14=ar.pfs,0,1,3,0 - srlz.d // serialize psr.i - add loc0=(8*8),r16 // remember where trapframe is - ;; -} { .mlx - mov out0=r15 // syscall number (from user) - movl gp=__gp // kernel globals -} { .mmb - mov out1=r16 // arguments - add out2=(8*8),r16 // trapframe pointer - br.call.sptk.many rp=syscall // do the work -} { .mmi -3: rsm psr.i // we know that psr.i == 1 - add r14=PC_CURTHREAD,r13 // &curthread - nop.i 0 - ;; -} { .mmi - ld8 r14=[r14] // curthread - ;; - nop.i 0 - ;; -} { .mmi - ;; - add r14=TD_FLAGS,r14 // &curthread->td_flags - nop.i 0 - ;; -} { .mmi - ld4 r14=[r14] // curkse->ke_flags - ;; - nop.m 0 - tbit.nz p6,p7=r14,11 // TDF_ASTPENDING - ;; -} { .mib - nop.m 0 -(p7) tbit.nz.or.andcm p6,p7=r14,16 // TDF_NEEDRESCHED -(p7) br.cond.dptk 2f - ;; -} { .mmi - ssm psr.i // restore interrupts - ;; - srlz.d - mov out0=loc0 // trapframe argument to ast() -} { .mib - nop.m 0 - nop.i 0 - br.call.sptk.many rp=ast -} { .mib - nop.m 0 - nop.i 0 - br 3b -} { .mii -2: ld8 r14=[loc0] // check tf_flags - dep r15=0,loc0,61,3 // physical address of trapframe - ;; - tbit.z p6,p0=r14,0 // check FRAME_SYSCALL bit - ;; -} { .mib -(p6) add sp=-16,loc0 // do a full restore if clear - add r16=SIZEOF_TRAPFRAME,loc0 // new kernel sp -(p6) br.dpnt.many exception_restore -} { .mmi - rsm psr.dt|psr.ic|psr.i // get ready to restore - ;; - srlz.i // serialise psr.dt and psr.ic - add r30=TF_R_R15,r15 // point at r15 - ;; -} { .mmi - alloc r14=ar.pfs,0,0,0,0 // discard register frame - mov ar.k6=r16 // restore kernel sp - add r31=TF_R_SP,r15 // point at sp - ;; -} { .mmi - ld8 r15=[r30],TF_R_R10-TF_R_R15 // restore r15, skip to r10 - ld8 sp=[r31],TF_R_R9-TF_R_SP // restore user sp, skip to r9 - nop.i 0 - ;; -} { .mmi - ld8 r10=[r30],-16 // restore r10, skip to r8 - ld8 r9=[r31],TF_R_R1-TF_R_R9 // restore r9, skip to r1 - nop.i 0 - ;; -} { .mmi - ld8 r8=[r30],TF_B-TF_R_R8 // restore r8, skip to b0 - ld8 r1=[r31],TF_AR_FPSR-TF_R_R1 // restore r1, skip to ar.fpsr - nop.i 0 - ;; -} { .mmi - ld8 r16=[r30],-32 // restore b0, skip to ar.ccv - ld8 r17=[r31],-16 // restore ar.fpsr, skip to ar.unat - nop.i 0 - ;; -} { .mmi - ld8 r18=[r30],-16 // restore ar.ccv, skip to ndirty - ld8 r19=[r31],-16 // restore ar.unat, skip to ar.rnat - mov b0=r16 - ;; -} { .mmi - ld8 r20=[r30],-16 // restore ndirty, skip to ar.bspstore - ld8 r21=[r31],-16 // restore ar.rnat, skip to cr.ifs - nop.i 0 - ;; -} { .mmi - ld8 r16=[r30],-16 // restore ar.bspstore, skip to ar.pfs - mov ar.fpsr=r17 - shl r20=r20,16 // value for ar.rsc - ;; -} { .mmi - ld8 r22=[r31],-16 // restore cr.ifs, skip to ar.rsc - mov.m ar.ccv=r18 - nop.i 0 - ;; -} { .mmi - ld8 r17=[r30],-16 // restore ar.pfs, skip to pr - mov.m ar.unat=r19 - nop.i 0 - ;; -} { .mmi - ld8 r18=[r31],-32 // restore ar.rsc, skip to cr.ipsr - mov.m ar.rsc=r20 // setup for loadrs - nop.i 0 - ;; -} { .mmi - loadrs // restore user stacked registers - ;; - mov.m ar.bspstore=r16 // back to user backing store - mov.i ar.pfs=r17 - ;; -} { .mmi - mov.m ar.rnat=r21 - mov.m ar.rsc=r18 - nop.i 0 - ;; -} { .mmi - ld8 r16=[r30],-32 // restore pr, skip to cr.iip - ld8 r17=[r31] // restore cr.ipsr - nop.i 0 - ;; -} { .mmi - ld8 r18=[r30] // restore cr.iip - mov cr.ifs=r22 - nop.i 0 - ;; -} { .mmi - mov cr.iip=r18 - mov cr.ipsr=r17 - mov pr=r16,0x1ffff - ;; -} { .bbb - rfi -} - - // This is done as a function call to make sure that we only - // have output registers in the register frame. It also gives - // us a chance to use alloc to round up to 8 arguments for - // simplicity. - // - // We are still running in physical mode with psr.ic==0 because - // we haven't yet covered the user's register frame to get a - // value for cr.ifs -Lsaveargs: -{ .mii - alloc r14=ar.pfs,0,0,8,0 // round up to 8 outputs - extr.u r31=sp,0,61 // physical address - ;; - add r30=8,r31 - ;; -} { .mmi - st8 [r31]=r32,16 - st8 [r30]=r33,16 - ;; -} { .mmi - st8 [r31]=r34,16 - st8 [r30]=r35,16 - ;; -} { .mmi - st8 [r31]=r36,16 - st8 [r30]=r37,16 - ;; -} { .mmb - st8 [r31]=r38 - st8 [r30]=r39 - br.ret.sptk.many b0 -} - .global do_syscall_end -do_syscall_end: - -END(do_syscall) diff --git a/sys/ia64/ia64/exception.s b/sys/ia64/ia64/exception.s index 9e89aaa..947df2d 100644 --- a/sys/ia64/ia64/exception.s +++ b/sys/ia64/ia64/exception.s @@ -1,4 +1,5 @@ /*- + * Copyright (c) 2003 Marcel Moolenaar * Copyright (c) 2000 Doug Rabson * All rights reserved. * @@ -31,12 +32,541 @@ #include <assym.s> /* - * ar.k7 = curthread - * ar.k6 = ksp - * ar.k5 = kbsp - * ar.k4 = pcpup + * ar.k7 = kernel memory stack + * ar.k6 = kernel register stack + * ar.k5 = EPC gateway page + * ar.k4 = PCPU data */ + .text + +/* + * exception_save: save interrupted state + * + * Arguments: + * r16 address of bundle that contains the branch. The + * return address will be the next bundle. + * r17 the value to save as ifa in the trapframe. This + * normally is cr.ifa, but some interruptions set + * set cr.iim and not cr.ifa. + * + * Returns: + * p15 interrupted from user stack + * p14 interrupted from kernel stack + * p13 interrupted from user backing store + * p12 interrupted from kernel backing store + * p11 interrupts were enabled + * p10 interrupts were disabled + */ +ENTRY(exception_save, 0) +{ .mii + mov r20=ar.unat + extr.u r31=sp,61,3 + mov r18=pr + ;; +} +{ .mmi + cmp.le p14,p15=5,r31 + ;; +(p15) mov r23=ar.k7 // kernel memory stack +(p14) mov r23=sp + ;; +} +{ .mii + mov r21=ar.rsc + add r30=-SIZEOF_TRAPFRAME,r23 + ;; + dep r30=0,r30,0,10 + ;; +} +{ .mmi + mov ar.rsc=0 + sub r19=r23,r30 + add r31=8,r30 + ;; +} +{ .mlx + mov r22=cr.iip + movl r26=exception_save_restart + ;; +} + + /* + * We have a 1KB aligned trapframe, pointed to by sp. If we write + * to the trapframe, we may trigger a data nested TLB fault. By + * aligning the trapframe on a 1KB boundary, we guarantee that if + * we get a data nested TLB fault, it will be on the very first + * write. Since the data nested TLB fault does not preserve any + * state, we have to be careful what we clobber. Consequently, we + * have to be careful what we use here. Below a list of registers + * that are currently alive: + * r16,r17=arguments + * r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS + * r26=restart point + * r30,r31=trapframe pointers + * p14,p15=memory stack switch + */ +exception_save_restart: +{ .mmi + st8 [r30]=r19,16 // length + st8 [r31]=r0,16 // flags + add r19=16,r19 + ;; +} +{ .mmi + st8.spill [r30]=sp,16 // sp + st8 [r31]=r20,16 // unat + sub sp=r23,r19 + ;; +} +{ .mmi + mov r19=ar.rnat + mov r20=ar.bspstore + mov r23=rp + ;; +} + // r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp +{ .mmi + st8 [r30]=r23,16 // rp + st8 [r31]=r18,16 // pr + mov r24=ar.pfs + ;; +} +{ .mmb + st8 [r30]=r24,16 // pfs + st8 [r31]=r20,16 // bspstore + cover + ;; +} +{ .mmi + mov r18=ar.fpsr + mov r23=cr.ipsr + extr.u r24=r20,61,3 + ;; +} + // r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr +{ .mmi + st8 [r30]=r19,16 // rnat + st8 [r31]=r0,16 // __spare + cmp.le p12,p13=5,r24 + ;; +} +{ .mmi + st8.spill [r30]=r13,16 // tp + st8 [r31]=r21,16 // rsc + tbit.nz p11,p10=r23,14 // p11=interrupts enabled + ;; +} +{ .mmi + st8 [r30]=r18,16 // fpsr +(p13) mov r20=ar.k6 // kernel register stack + nop 0 + ;; +} + // r20=bspstore, r22=iip, r23=ipsr +{ .mmi + st8 [r31]=r23,16 // psr + mov ar.bspstore=r20 + nop 0 + ;; +} +{ .mmi + mov r18=ar.bsp + ;; + mov r19=cr.ifs + sub r18=r18,r20 + ;; +} +{ .mmi + st8.spill [r30]=gp,16 // gp + st8 [r31]=r18,16 // ndirty + nop 0 + ;; +} + // r19=ifs, r22=iip +{ .mmi + st8 [r30]=r19,16 // cfm + st8 [r31]=r22,16 // iip + nop 0 + ;; +} +{ .mmi + st8 [r30]=r17 // ifa + mov r18=cr.isr + add r29=16,r30 + ;; +} +{ .mmi + st8 [r31]=r18 // isr + add r30=8,r29 + add r31=16,r29 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r2,16 // r2 + .mem.offset 8,0 + st8.spill [r31]=r3,16 // r3 + add r2=9*8,r29 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r8,16 // r8 + .mem.offset 8,0 + st8.spill [r31]=r9,16 // r9 + add r3=8,r2 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r10,16 // r10 + .mem.offset 8,0 + st8.spill [r31]=r11,16 // r11 + add r8=16,r16 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r30]=r14 // r14 + .mem.offset 8,0 + st8.spill [r31]=r15 // r15 + mov r9=r29 +} +{ .mmb + mov r10=ar.csd + mov r11=ar.ssd + bsw.1 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r16,16 // r16 + .mem.offset 8,0 + st8.spill [r3]=r17,16 // r17 + mov r14=b6 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r18,16 // r18 + .mem.offset 8,0 + st8.spill [r3]=r19,16 // r19 + mov r15=b7 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r20,16 // r20 + .mem.offset 8,0 + st8.spill [r3]=r21,16 // r21 + mov b7=r8 + ;; +} +{ .mmi + .mem.offset 0,0 + st8.spill [r2]=r22,16 // r22 + .mem.offset 8,0 + st8.spill [r3]=r23,16 // r23 + ;; +} + + .mem.offset 0,0 + st8.spill [r2]=r24,16 // r24 + .mem.offset 8,0 + st8.spill [r3]=r25,16 // r25 + ;; + .mem.offset 0,0 + st8.spill [r2]=r26,16 // r26 + .mem.offset 8,0 + st8.spill [r3]=r27,16 // r27 + ;; + .mem.offset 0,0 + st8.spill [r2]=r28,16 // r28 + .mem.offset 8,0 + st8.spill [r3]=r29,16 // r29 + ;; + .mem.offset 0,0 + st8.spill [r2]=r30,16 // r30 + .mem.offset 8,0 + st8.spill [r3]=r31,16 // r31 + ;; + +{ .mmi + st8 [r2]=r14,16 // b6 + mov r17=ar.unat + nop 0 + ;; +} +{ .mmi + st8 [r3]=r15,16 // b7 + mov r16=ar.ccv + nop 0 + ;; +} +{ .mmi + st8 [r2]=r16,16 // ccv + st8 [r3]=r10,16 // csd + nop 0 + ;; +} +{ .mmi + st8 [r2]=r11,24 // ssd + st8 [r9]=r17 + nop 0 + ;; +} + + stf.spill [r3]=f6,32 // f6 + stf.spill [r2]=f7,32 // f7 + ;; + stf.spill [r3]=f8,32 // f8 + stf.spill [r2]=f9,32 // f9 + ;; + stf.spill [r3]=f10,32 // f10 + stf.spill [r2]=f11,32 // f11 + ;; + stf.spill [r3]=f12,32 // f12 + stf.spill [r2]=f13,32 // f13 + ;; + stf.spill [r3]=f14 // f14 + stf.spill [r2]=f15 // f15 + ;; +{ .mmi + mov ar.rsc=3 + mov r13=ar.k4 + nop 0 + ;; +} +{ .mlx + ssm psr.ic|psr.dfh + movl gp=__gp + ;; +} +{ .mfb + srlz.d + nop 0 + br.sptk b7 + ;; +} +END(exception_save) + +/* + * exception_restore: restore interrupted state + * + * Arguments: + * sp+16 trapframe pointer + */ +ENTRY(exception_restore, 0) +{ .mmi + rsm psr.i + add sp=16,sp + nop 0 + ;; +} +{ .mmi + add r3=SIZEOF_TRAPFRAME-32,sp + add r2=SIZEOF_TRAPFRAME-16,sp + add r8=SIZEOF_SPECIAL+16,sp + ;; +} + // The next load can trap. Let it be... + ldf.fill f15=[r2],-32 // f15 + ldf.fill f14=[r3],-32 // f14 + ;; + ldf.fill f13=[r2],-32 // f13 + ldf.fill f12=[r3],-32 // f12 + ;; + ldf.fill f11=[r2],-32 // f11 + ldf.fill f10=[r3],-32 // f10 + ;; + ldf.fill f9=[r2],-32 // f9 + ldf.fill f8=[r3],-32 // f8 + ;; + ldf.fill f7=[r2],-24 // f7 + ldf.fill f6=[r3],-16 // f6 + ;; + +{ .mmi + ld8 r8=[r8] // unat (after) + ;; + mov ar.unat=r8 + nop 0 + ;; +} + + ld8 r10=[r2],-16 // ssd + ld8 r11=[r3],-16 // csd + ;; + mov ar.ssd=r10 + mov ar.csd=r11 + + ld8 r14=[r2],-16 // ccv + ld8 r15=[r3],-16 // b7 + ;; + +{ .mmi + mov ar.ccv=r14 + ld8 r8=[r2],-16 // b6 + mov b7=r15 + ;; +} +{ .mmi + ld8.fill r31=[r3],-16 // r31 + ld8.fill r30=[r2],-16 // r30 + mov b6=r8 + ;; +} + + ld8.fill r29=[r3],-16 // r29 + ld8.fill r28=[r2],-16 // r28 + ;; + ld8.fill r27=[r3],-16 // r27 + ld8.fill r26=[r2],-16 // r26 + ;; + ld8.fill r25=[r3],-16 // r25 + ld8.fill r24=[r2],-16 // r24 + ;; + ld8.fill r23=[r3],-16 // r23 + ld8.fill r22=[r2],-16 // r22 + ;; + ld8.fill r21=[r3],-16 // r21 + ld8.fill r20=[r2],-16 // r20 + ;; + ld8.fill r19=[r3],-16 // r19 + ld8.fill r18=[r2],-16 // r18 + ;; + +{ .mmb + ld8.fill r17=[r3],-16 // r17 + ld8.fill r16=[r2],-16 // r16 + bsw.0 + ;; +} +{ .mmi + ld8.fill r15=[r3],-16 // r15 + ld8.fill r14=[r2],-16 // r14 + add r31=16,sp + ;; +} +{ .mmi + ld8 r16=[sp] // tf_length + ld8.fill r11=[r3],-16 // r11 + add r30=24,sp + ;; +} +{ .mmi + ld8.fill r10=[r2],-16 // r10 + ld8.fill r9=[r3],-16 // r9 + add r16=r16,sp // ar.k7 + ;; +} +{ .mmi + ld8.fill r8=[r2],-16 // r8 + ld8.fill r3=[r3] // r3 + ;; +} + // We want nested TLB faults from here on... + rsm psr.ic|psr.i + ld8.fill r2=[r2] // r2 + nop 0 + ;; + srlz.d + ld8.fill sp=[r31],16 // sp + nop 0 + ;; + + ld8 r17=[r30],16 // unat + ld8 r29=[r31],16 // rp + ;; + ld8 r18=[r30],16 // pr + ld8 r28=[r31],16 // pfs + mov rp=r29 + ;; + ld8 r20=[r30],24 // bspstore + ld8 r21=[r31],24 // rnat + mov ar.pfs=r28 + ;; + ld8.fill r29=[r30],16 // tp + ld8 r22=[r31],16 // rsc + ;; +{ .mmi + ld8 r23=[r30],16 // fpsr + ld8 r24=[r31],16 // psr + extr.u r28=r20,61,3 + ;; +} +{ .mmi + ld8.fill r1=[r30],16 // gp + ld8 r25=[r31],16 // ndirty + cmp.le p14,p15=5,r28 + ;; +} +{ .mmb + ld8 r26=[r30] // cfm + ld8 r19=[r31] // ip +(p14) br.cond.sptk 1f + ;; +} +{ .mib + // Switch register stack + alloc r30=ar.pfs,0,0,0,0 // discard current frame + shl r31=r25,16 // value for ar.rsc + nop 0 + ;; +} + // The loadrs can fault if the backing store is not currently + // mapped. We assured forward progress by getting everything we + // need from the trapframe so that we don't care if the CPU + // purges that translation when it needs to insert a new one for + // the backing store. +{ .mmi + mov ar.rsc=r31 // setup for loadrs + mov ar.k7=r16 + mov r13=r29 + ;; +} +exception_restore_restart: +{ .mmi + mov r30=ar.bspstore + ;; + loadrs // load user regs + nop 0 + ;; +} +{ .mmi + mov r31=ar.bspstore + ;; + mov ar.bspstore=r20 + dep r31=0,r31,0,9 + ;; +} +{ .mmb + mov ar.k6=r31 + mov ar.rnat=r21 + nop 0 + ;; +} +1: +{ .mmb + mov ar.unat=r17 + mov cr.iip=r19 + nop 0 +} +{ .mmi + mov cr.ipsr=r24 + mov cr.ifs=r26 + mov pr=r18,0x1fffe + ;; +} +{ .mmb + mov ar.rsc=r22 + mov ar.fpsr=r23 + rfi + ;; +} +END(exception_restore) + /* * Call exception_save_regs to preserve the interrupted state in a * trapframe. Note that we don't use a call instruction because we @@ -46,20 +576,27 @@ * which will restore the interrupted state before executing an rfi to * resume it. */ -#define TRAP(_n_) \ -1: mov r17=ip;; \ - add r17=2f-1b,r17; \ - mov r16=b6;; \ - mov b6=r17; \ - br.sptk.few exception_save; \ -2: (p3) ssm psr.i; \ - alloc r15=ar.pfs,0,0,3,0; \ - mov out0=_n_; \ - mov out1=r14; \ - mov out2=sp;; \ - add sp=-16,sp;; \ - br.call.sptk.few rp=trap; \ -3: br.sptk.many exception_restore +#define TRAP(_n_, _ifa_) \ +{ .mib ; \ + mov r17=_ifa_ ; \ + mov r16=ip ; \ + br.sptk exception_save ; \ +} ; \ +{ .mmi ; \ +(p11) ssm psr.i ;; \ + alloc r15=ar.pfs,0,0,2,0 ; \ + mov out0=_n_ ;; \ +} ; \ +{ .mfb ; \ + add out1=16,sp ; \ + nop 0 ; \ + br.call.sptk rp=trap ; \ +} ; \ +{ .mfb ; \ + nop 0 ; \ + nop 0 ; \ + br.sptk exception_restore ; \ +} #define IVT_ENTRY(name, offset) \ .org ia64_vector_table + offset; \ @@ -88,7 +625,7 @@ ivt_##name: ia64_vector_table: IVT_ENTRY(VHPT_Translation, 0x0000) - TRAP(0) + TRAP(0, cr.ifa) IVT_END(VHPT_Translation) IVT_ENTRY(Instruction_TLB, 0x0400) @@ -103,8 +640,8 @@ IVT_ENTRY(Instruction_TLB, 0x0400) ;; ld8 r21=[r21] // check VHPT tag ;; - cmp.ne p1,p0=r21,r19 -(p1) br.dpnt.few 1f + cmp.ne p15,p0=r21,r19 +(p15) br.dpnt.few 1f ;; ld8 r21=[r18] // read pte mov pr=r17,0x1ffff @@ -118,15 +655,15 @@ IVT_ENTRY(Instruction_TLB, 0x0400) ;; srlz.d // serialize ;; -2: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +2: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 3f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 3f // if not, read next in chain ;; ld8 r21=[r20],8 // read pte ;; @@ -165,7 +702,7 @@ IVT_ENTRY(Instruction_TLB, 0x0400) ;; srlz.d ;; - TRAP(20) // Page Not Present trap + TRAP(20, cr.ifa) // Page Not Present trap IVT_END(Instruction_TLB) IVT_ENTRY(Data_TLB, 0x0800) @@ -180,8 +717,8 @@ IVT_ENTRY(Data_TLB, 0x0800) ;; ld8 r21=[r21] // check VHPT tag ;; - cmp.ne p1,p0=r21,r19 -(p1) br.dpnt.few 1f + cmp.ne p15,p0=r21,r19 +(p15) br.dpnt.few 1f ;; ld8 r21=[r18] // read pte mov pr=r17,0x1ffff @@ -195,15 +732,15 @@ IVT_ENTRY(Data_TLB, 0x0800) ;; srlz.d // serialize ;; -2: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +2: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 3f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 3f // if not, read next in chain ;; ld8 r21=[r20],8 // read pte ;; @@ -242,7 +779,7 @@ IVT_ENTRY(Data_TLB, 0x0800) ;; srlz.d ;; - TRAP(20) // Page Not Present trap + TRAP(20, cr.ifa) // Page Not Present trap IVT_END(Data_TLB) IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) @@ -251,12 +788,12 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) ;; extr.u r17=r16,61,3 // get region number ;; - cmp.ge p3,p0=5,r17 // RR0-RR5? - cmp.eq p1,p2=7,r17 // RR7->p1, RR6->p2 -(p3) br.spnt 9f + cmp.ge p13,p0=5,r17 // RR0-RR5? + cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 +(p13) br.spnt 9f ;; -(p1) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX -(p2) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX +(p15) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX +(p14) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RX ;; dep r16=0,r16,50,14 // clear bits above PPN ;; @@ -267,7 +804,7 @@ IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) ;; rfi 9: mov pr=r18,0x1ffff // restore predicates - TRAP(3) + TRAP(3, cr.ifa) IVT_END(Alternate_Instruction_TLB) IVT_ENTRY(Alternate_Data_TLB, 0x1000) @@ -276,12 +813,12 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000) ;; extr.u r17=r16,61,3 // get region number ;; - cmp.ge p3,p0=5,r17 // RR0-RR5? - cmp.eq p1,p2=7,r17 // RR7->p1, RR6->p2 -(p3) br.spnt 9f + cmp.ge p13,p0=5,r17 // RR0-RR5? + cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 +(p13) br.spnt 9f ;; -(p1) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW -(p2) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW +(p15) movl r17=PTE_P+PTE_MA_WB+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW +(p14) movl r17=PTE_P+PTE_MA_UC+PTE_A+PTE_D+PTE_PL_KERN+PTE_AR_RW ;; dep r16=0,r16,50,14 // clear bits above PPN ;; @@ -292,19 +829,92 @@ IVT_ENTRY(Alternate_Data_TLB, 0x1000) ;; rfi 9: mov pr=r18,0x1ffff // restore predicates - TRAP(4) + TRAP(4, cr.ifa) IVT_END(Alternate_Data_TLB) IVT_ENTRY(Data_Nested_TLB, 0x1400) - TRAP(5) + // See exception_save_restart and exception_restore_restart for the + // contexts that may cause a data nested TLB. We can only use the + // banked general registers and predicates, but don't use: + // p14 & p15 - Set in exception save + // r16 & r17 - Arguments to exception save + // r30 - Faulting address (modulo page size) + // We assume r30 has the virtual addresses that relate to the data + // nested TLB fault. The address does not have to be exact, as long + // as it's in the same page. We use physical addressing to avoid + // double nested faults. Since all virtual addresses we encounter + // here are direct mapped region 7 addresses, we have no problem + // constructing physical addresses. +{ .mlx + rsm psr.dt + movl r27=ia64_kptdir + ;; +} +{ .mii + srlz.d + dep r27=0,r27,61,3 + extr.u r28=r30,PAGE_SHIFT,61-PAGE_SHIFT + ;; +} +{ .mii + ld8 r27=[r27] + shr.u r29=r28,PAGE_SHIFT-5 // dir index + extr.u r28=r28,0,PAGE_SHIFT-5 // pte index + ;; +} +{ .mmi + shladd r27=r29,3,r27 + ;; + mov cr.ifa=r30 + dep r27=0,r27,61,3 + ;; +} +{ .mmi + ld8 r27=[r27] + mov r29=rr[r30] + shl r28=r28,5 + ;; +} +{ .mii + add r27=r27,r28 // address of pte + dep r29=0,r29,0,2 + ;; + dep r27=0,r27,61,3 + ;; +} +{ .mmi + ld8 r28=[r27] + ;; + mov cr.itir=r29 + or r28=PTE_D|PTE_A,r28 + ;; +} +{ .mlx + st8 [r27]=r28 + movl r29=exception_save_restart + ;; +} +{ .mmi + itc.d r28 + ;; + ssm psr.dt + cmp.eq p12,p13=r26,r29 + ;; +} +{ .mbb + srlz.d +(p12) br.sptk exception_save_restart +(p13) br.sptk exception_restore_restart + ;; +} IVT_END(Data_Nested_TLB) IVT_ENTRY(Instruction_Key_Miss, 0x1800) - TRAP(6) + TRAP(6, cr.ifa) IVT_END(Instruction_Key_Miss) IVT_ENTRY(Data_Key_Miss, 0x1c00) - TRAP(7) + TRAP(7, cr.ifa) IVT_END(Data_Key_Miss) IVT_ENTRY(Dirty_Bit, 0x2000) @@ -325,15 +935,15 @@ IVT_ENTRY(Dirty_Bit, 0x2000) ;; srlz.d // serialize ;; -1: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +1: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 2f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_D|PTE_A @@ -373,7 +983,7 @@ IVT_ENTRY(Dirty_Bit, 0x2000) br.cond.sptk.few 1b // loop 9: mov pr=r17,0x1ffff // restore predicates - TRAP(8) // die horribly + TRAP(8, cr.ifa) // die horribly IVT_END(Dirty_Bit) IVT_ENTRY(Instruction_Access_Bit, 0x2400) @@ -394,15 +1004,15 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400) ;; srlz.d // serialize ;; -1: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +1: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 2f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_A @@ -442,7 +1052,7 @@ IVT_ENTRY(Instruction_Access_Bit, 0x2400) br.cond.sptk.few 1b // loop 9: mov pr=r17,0x1ffff // restore predicates - TRAP(9) + TRAP(9, cr.ifa) IVT_END(Instruction_Access_Bit) IVT_ENTRY(Data_Access_Bit, 0x2800) @@ -463,15 +1073,15 @@ IVT_ENTRY(Data_Access_Bit, 0x2800) ;; srlz.d // serialize ;; -1: cmp.eq p1,p0=r0,r20 // done? -(p1) br.cond.spnt.few 9f // bail if done +1: cmp.eq p15,p0=r0,r20 // done? +(p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; - cmp.ne p1,p0=r21,r19 // compare tags -(p1) br.cond.sptk.few 2f // if not, read next in chain + cmp.ne p15,p0=r21,r19 // compare tags +(p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_A @@ -511,1344 +1121,283 @@ IVT_ENTRY(Data_Access_Bit, 0x2800) br.cond.sptk.few 1b // loop 9: mov pr=r17,0x1ffff // restore predicates - TRAP(10) + TRAP(10, cr.ifa) IVT_END(Data_Access_Bit) IVT_ENTRY(Break_Instruction, 0x2c00) - mov r16=pr // save pr for a moment - mov r17=cr.iim;; // read break value - mov r18=0x100000;; // syscall number - cmp.ne p6,p0=r18,r17;; // check for syscall -(p6) br.dpnt.few 9f - - mov r17=cr.ipsr;; // check for user mode - extr.u r17=r17,32,2;; - cmp.eq p6,p0=r0,r17 -(p6) br.dpnt.few 9f // trap if kernel mode - - // Note: p6 and p7 are temporaries so we don't need to restore - // the value of pr here since the user-mode program assumes - // that syscalls only preserve the function-preserved state. - - br.sptk.many do_syscall +{ .mib + mov r17=cr.iim + mov r16=ip + br.sptk exception_save + ;; +} +{ .mmi + alloc r15=ar.pfs,0,0,2,0 + flushrs + mov out0=11 + ;; +} +{ .mib +(p11) ssm psr.i + add out1=16,sp + br.call.sptk rp=trap ;; -9: mov pr=r16,0x1ffff // restore pr - TRAP(11) +} +{ .mfb + nop 0 + nop 0 + br.sptk exception_restore + ;; +} IVT_END(Break_Instruction) IVT_ENTRY(External_Interrupt, 0x3000) - mov r16=b6 // save user's b6 -1: mov r17=ip;; // construct return address - add r17=2f-1b,r17;; // for exception_save - mov b6=r17 - br.sptk.few exception_save // 'call' exception_save - +{ .mib + mov r17=cr.lid // cr.iim and cr.ifa are undefined. + mov r16=ip + br.sptk exception_save + ;; +} +{ .mmb 2: alloc r14=ar.pfs,0,0,2,0 // make a frame for calling with - - mov out1=sp;; - add sp=-16,sp;; + add out1=16,sp + nop 0 +} 3: mov out0=cr.ivr // find interrupt vector ;; - cmp.eq p6,p0=15,out0 // check for spurious vector number -(p6) br.dpnt.few exception_restore // if spurious, we are done + cmp.eq p15,p0=15,out0 // check for spurious vector number +(p15) br.dpnt.few exception_restore // if spurious, we are done ;; ssm psr.i // re-enable interrupts - ;; // now that we are in-progress - srlz.d - ;; br.call.sptk.many rp=interrupt // call high-level handler - + ;; rsm psr.i // disable interrupts ;; srlz.d - ;; mov cr.eoi=r0 // and ack the interrupt ;; srlz.d br.sptk.few 3b // loop for more + ;; IVT_END(External_Interrupt) IVT_ENTRY(Reserved_3400, 0x3400) - TRAP(13) + TRAP(13, cr.ifa) IVT_END(Reserved_3400) IVT_ENTRY(Reserved_3800, 0x3800) - TRAP(14) + TRAP(14, cr.ifa) IVT_END(Reserved_3800) IVT_ENTRY(Reserved_3c00, 0x3c00) - TRAP(15) + TRAP(15, cr.ifa) IVT_END(Reserved_3c00) IVT_ENTRY(Reserved_4000, 0x4000) - TRAP(16) + TRAP(16, cr.ifa) IVT_END(Reserved_4000) IVT_ENTRY(Reserved_4400, 0x4400) - TRAP(17) + TRAP(17, cr.ifa) IVT_END(Reserved_4400) IVT_ENTRY(Reserved_4800, 0x4800) - TRAP(18) + TRAP(18, cr.ifa) IVT_END(Reserved_4800) IVT_ENTRY(Reserved_4c00, 0x4c00) - TRAP(19) + TRAP(19, cr.ifa) IVT_END(Reserved_4c00) IVT_ENTRY(Page_Not_Present, 0x5000) - TRAP(20) + TRAP(20, cr.ifa) IVT_END(Page_Not_Present) IVT_ENTRY(Key_Permission, 0x5100) - TRAP(21) + TRAP(21, cr.ifa) IVT_END(Key_Permission) IVT_ENTRY(Instruction_Access_Rights, 0x5200) - TRAP(22) + TRAP(22, cr.ifa) IVT_END(Instruction_Access_Rights) IVT_ENTRY(Data_Access_Rights, 0x5300) - TRAP(23) + TRAP(23, cr.ifa) IVT_END(Data_Access_Rights) IVT_ENTRY(General_Exception, 0x5400) - TRAP(24) + TRAP(24, cr.ifa) IVT_END(General_Exception) IVT_ENTRY(Disabled_FP_Register, 0x5500) - TRAP(25) + TRAP(25, cr.ifa) IVT_END(Disabled_FP_Register) IVT_ENTRY(NaT_Consumption, 0x5600) - TRAP(26) + TRAP(26, cr.ifa) IVT_END(NaT_Consumption) IVT_ENTRY(Speculation, 0x5700) - TRAP(27) + TRAP(27, cr.iim) IVT_END(Speculation) IVT_ENTRY(Reserved_5800, 0x5800) - TRAP(28) + TRAP(28, cr.ifa) IVT_END(Reserved_5800) IVT_ENTRY(Debug, 0x5900) - TRAP(29) + TRAP(29, cr.ifa) IVT_END(Debug) IVT_ENTRY(Unaligned_Reference, 0x5a00) - TRAP(30) + TRAP(30, cr.ifa) IVT_END(Unaligned_Reference) IVT_ENTRY(Unsupported_Data_Reference, 0x5b00) - TRAP(31) + TRAP(31, cr.ifa) IVT_END(Unsupported_Data_Reference) IVT_ENTRY(Floating_Point_Fault, 0x5c00) - TRAP(32) + TRAP(32, cr.ifa) IVT_END(Floating_Point_Fault) IVT_ENTRY(Floating_Point_Trap, 0x5d00) - TRAP(33) + TRAP(33, cr.ifa) IVT_END(Floating_Point_Trap) IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00) - TRAP(34) + TRAP(34, cr.ifa) IVT_END(Lower_Privilege_Transfer_Trap) IVT_ENTRY(Taken_Branch_Trap, 0x5f00) - TRAP(35) + TRAP(35, cr.ifa) IVT_END(Taken_Branch_Trap) IVT_ENTRY(Single_Step_Trap, 0x6000) - TRAP(36) + TRAP(36, cr.ifa) IVT_END(Single_Step_Trap) IVT_ENTRY(Reserved_6100, 0x6100) - TRAP(37) + TRAP(37, cr.ifa) IVT_END(Reserved_6100) IVT_ENTRY(Reserved_6200, 0x6200) - TRAP(38) + TRAP(38, cr.ifa) IVT_END(Reserved_6200) IVT_ENTRY(Reserved_6300, 0x6300) - TRAP(39) + TRAP(39, cr.ifa) IVT_END(Reserved_6300) IVT_ENTRY(Reserved_6400, 0x6400) - TRAP(40) + TRAP(40, cr.ifa) IVT_END(Reserved_6400) IVT_ENTRY(Reserved_6500, 0x6500) - TRAP(41) + TRAP(41, cr.ifa) IVT_END(Reserved_6500) IVT_ENTRY(Reserved_6600, 0x6600) - TRAP(42) + TRAP(42, cr.ifa) IVT_END(Reserved_6600) IVT_ENTRY(Reserved_6700, 0x6700) - TRAP(43) + TRAP(43, cr.ifa) IVT_END(Reserved_6700) IVT_ENTRY(Reserved_6800, 0x6800) - TRAP(44) + TRAP(44, cr.ifa) IVT_END(Reserved_6800) IVT_ENTRY(IA_32_Exception, 0x6900) - TRAP(45) + TRAP(45, cr.ifa) IVT_END(IA_32_Exception) IVT_ENTRY(IA_32_Intercept, 0x6a00) - TRAP(46) + TRAP(46, cr.iim) IVT_END(IA_32_Intercept) IVT_ENTRY(IA_32_Interrupt, 0x6b00) - TRAP(47) + TRAP(47, cr.ifa) IVT_END(IA_32_Interrupt) IVT_ENTRY(Reserved_6c00, 0x6c00) - TRAP(48) + TRAP(48, cr.ifa) IVT_END(Reserved_6c00) IVT_ENTRY(Reserved_6d00, 0x6d00) - TRAP(49) + TRAP(49, cr.ifa) IVT_END(Reserved_6d00) IVT_ENTRY(Reserved_6e00, 0x6e00) - TRAP(50) + TRAP(50, cr.ifa) IVT_END(Reserved_6e00) IVT_ENTRY(Reserved_6f00, 0x6f00) - TRAP(51) + TRAP(51, cr.ifa) IVT_END(Reserved_6f00) IVT_ENTRY(Reserved_7000, 0x7000) - TRAP(52) + TRAP(52, cr.ifa) IVT_END(Reserved_7000) IVT_ENTRY(Reserved_7100, 0x7100) - TRAP(53) + TRAP(53, cr.ifa) IVT_END(Reserved_7100) IVT_ENTRY(Reserved_7200, 0x7200) - TRAP(54) + TRAP(54, cr.ifa) IVT_END(Reserved_7200) IVT_ENTRY(Reserved_7300, 0x7300) - TRAP(55) + TRAP(55, cr.ifa) IVT_END(Reserved_7300) IVT_ENTRY(Reserved_7400, 0x7400) - TRAP(56) + TRAP(56, cr.ifa) IVT_END(Reserved_7400) IVT_ENTRY(Reserved_7500, 0x7500) - TRAP(57) + TRAP(57, cr.ifa) IVT_END(Reserved_7500) IVT_ENTRY(Reserved_7600, 0x7600) - TRAP(58) + TRAP(58, cr.ifa) IVT_END(Reserved_7600) IVT_ENTRY(Reserved_7700, 0x7700) - TRAP(59) + TRAP(59, cr.ifa) IVT_END(Reserved_7700) IVT_ENTRY(Reserved_7800, 0x7800) - TRAP(60) + TRAP(60, cr.ifa) IVT_END(Reserved_7800) IVT_ENTRY(Reserved_7900, 0x7900) - TRAP(61) + TRAP(61, cr.ifa) IVT_END(Reserved_7900) IVT_ENTRY(Reserved_7a00, 0x7a00) - TRAP(62) + TRAP(62, cr.ifa) IVT_END(Reserved_7a00) IVT_ENTRY(Reserved_7b00, 0x7b00) - TRAP(63) + TRAP(63, cr.ifa) IVT_END(Reserved_7b00) IVT_ENTRY(Reserved_7c00, 0x7c00) - TRAP(64) + TRAP(64, cr.ifa) IVT_END(Reserved_7c00) IVT_ENTRY(Reserved_7d00, 0x7d00) - TRAP(65) + TRAP(65, cr.ifa) IVT_END(Reserved_7d00) IVT_ENTRY(Reserved_7e00, 0x7e00) - TRAP(66) + TRAP(66, cr.ifa) IVT_END(Reserved_7e00) IVT_ENTRY(Reserved_7f00, 0x7f00) - TRAP(67) + TRAP(67, cr.ifa) IVT_END(Reserved_7f00) - - .text - -/* - * exception_restore: restore interrupted state - * - * Arguments: - * sp+16 trapframe pointer - * r4 ar.pfs before the alloc in TRAP() - * - */ -ENTRY(exception_restore, 0) -{ .mfi - alloc r14=ar.pfs,0,0,1,0 // in case we call ast() - nop 1 - add r3=TF_CR_IPSR+16,sp - ;; -} -{ .mmi - ld8 r30=[r3] // ipsr - ;; - nop 2 - extr.u r16=r30,32,2 // extract ipsr.cpl - ;; -} -{ .mfb - cmp.eq p1,p2=r0,r16 // test for return to kernel mode - nop 3 -(p1) br.cond.dpnt 2f // no ast check for returns to kernel -} -3: -{ .mmi - add r3=PC_CURTHREAD,r13 // &curthread - ;; - ld8 r3=[r3] // curthread - add r2=(TDF_ASTPENDING|TDF_NEEDRESCHED),r0 - ;; -} -{ .mmb - mov r15=psr // save interrupt enable status - nop 4 - ;; -} -{ .mmi - ;; - rsm psr.i // disable interrupts - add r3=TD_FLAGS,r3 // &curthread->td_flags - ;; -} -{ .mmi - ld4 r14=[r3] // fetch curthread->td_flags - ;; - and r14=r2,r14 // flags & (TDF_ASTPENDING|TDF_NEEDRESCHED) - nop 5 - ;; -} -{ .mfb - cmp4.eq p6,p7=r0,r14 // == 0 ? - nop 6 -(p6) br.cond.dptk 2f - ;; -} -{ .mmi - mov psr.l=r15 // restore interrups - ;; - srlz.d - add out0=16,sp // trapframe argument to ast() -} -{ .bbb - br.call.sptk.many rp=ast // note: p1, p2 preserved - br.sptk 3b - nop 7 -} -2: -{ .mmi - rsm psr.ic|psr.dt|psr.i // disable interrupt collection and vm - ;; - srlz.i - add r3=16,sp - ;; -} -{ .mmi -(p2) add r16=SIZEOF_TRAPFRAME+16,sp // restore ar.k6 (kernel sp) - ;; -(p2) mov ar.k6=r16 - dep r3=0,r3,61,3 // physical address - ;; -} -{ .mmi - add r1=SIZEOF_TRAPFRAME-16,r3 // r1=&tf_f[FRAME_F15] - ;; - ldf.fill f15=[r1],-32 // r1=&tf_f[FRAME_F13] - add r2=SIZEOF_TRAPFRAME-32,r3 // r2=&tf_f[FRAME_F14] - ;; -} -{ .mmb - ldf.fill f14=[r2],-32 // r2=&tf_f[FRAME_F12] - ldf.fill f13=[r1],-32 // r1=&tf_f[FRAME_F11] - nop 8 - ;; -} -{ .mmi - ldf.fill f12=[r2],-32 // r2=&tf_f[FRAME_F10] - ldf.fill f11=[r1],-32 // r1=&tf_f[FRAME_F9] - nop 9 - ;; -} -{ .mmb - ldf.fill f10=[r2],-32 // r2=&tf_f[FRAME_F8] - ldf.fill f9=[r1],-32 // r1=&tf_f[FRAME_F7] - nop 10 - ;; -} -{ .mmi - ldf.fill f8=[r2],-32 // r2=&tf_f[FRAME_F6] - ldf.fill f7=[r1],-24 // r1=&tf_r[FRAME_R31] - nop 11 - ;; -} -{ .mmb - ldf.fill f6=[r2],-16 // r2=&tf_r[FRAME_R30] - ld8.fill r31=[r1],-16 // r1=&tf_r[FRAME_R29] - nop 12 - ;; -} -{ .mmi - ld8.fill r30=[r2],-16 // r2=&tf_r[FRAME_R28] - ld8.fill r29=[r1],-16 // r1=&tf_r[FRAME_R27] - nop 13 - ;; -} -{ .mmb - ld8.fill r28=[r2],-16 // r2=&tf_r[FRAME_R26] - ld8.fill r27=[r1],-16 // r1=&tf_r[FRAME_R25] - nop 14 - ;; -} -{ .mmi - ld8.fill r26=[r2],-16 // r2=&tf_r[FRAME_R24] - ld8.fill r25=[r1],-16 // r1=&tf_r[FRAME_R23] - nop 15 - ;; -} -{ .mmb - ld8.fill r24=[r2],-16 // r2=&tf_r[FRAME_R22] - ld8.fill r23=[r1],-16 // r1=&tf_r[FRAME_R21] - nop 16 - ;; -} -{ .mmi - ld8.fill r22=[r2],-16 // r2=&tf_r[FRAME_R20] - ld8.fill r21=[r1],-16 // r1=&tf_r[FRAME_R19] - nop 17 - ;; -} -{ .mmb - ld8.fill r20=[r2],-16 // r2=&tf_r[FRAME_R18] - ld8.fill r19=[r1],-16 // r1=&tf_r[FRAME_R17] - nop 18 - ;; -} -{ .mmi - ld8.fill r18=[r2],-16 // r2=&tf_r[FRAME_R16] - ld8.fill r17=[r1],-16 // r1=&tf_r[FRAME_R15] - nop 19 - ;; -} -{ .mfb - ld8.fill r16=[r2],-16 // r2=&tf_r[FRAME_R14] - nop 20 - bsw.0 // switch to bank 0 - ;; -} -{ .mmi - ld8.fill r15=[r1],-16 // r1=&tf_r[FRAME_R13] - ld8.fill r14=[r2],-16 // r2=&tf_r[FRAME_R12] - nop 21 - ;; -} - // Don't restore r13 if returning to kernel -{ .mmi - .pred.rel.mutex p1,p2 -(p2) ld8.fill r13=[r1],-16 // r1=&tf_r[FRAME_R11] - ld8.fill r12=[r2],-16 // r2=&tf_r[FRAME_R10] -(p1) add r1=-16,r1 // r1=&tf_r[FRAME_R11] - ;; -} -{ .mmb - ld8.fill r11=[r1],-16 // r1=&tf_r[FRAME_R9] - ld8.fill r10=[r2],-16 // r2=&tf_r[FRAME_R8] - nop 22 - ;; -} -{ .mmi - ld8.fill r9=[r1],-16 // r1=&tf_r[FRAME_R7] - ld8.fill r8=[r2],-16 // r2=&tf_r[FRAME_R6] - nop 23 - ;; -} -{ .mmb - ld8.fill r7=[r1],-16 // r1=&tf_r[FRAME_R5] - ld8.fill r6=[r2],-16 // r2=&tf_r[FRAME_R4] - nop 24 - ;; -} -{ .mmi - ld8.fill r5=[r1],-16 // r1=&tf_r[FRAME_R3] - ld8.fill r4=[r2],-16 // r2=&tf_r[FRAME_R2] - nop 25 - ;; -} -{ .mmb - ld8.fill r3=[r1],-16 // r1=&tf_r[FRAME_R1] - ld8.fill r23=[r2],-16 // r2=&tf_b[7] - nop 26 - ;; -} -{ .mmi - ld8.fill r24=[r1],-16 // r1=&tf_b[6] - ld8 r16=[r2],-16 // r16=b7, r2=&tf_b[5] - nop 27 - ;; -} -{ .mmi - ld8 r17=[r1],-16 // r17=b6, r1=&tf_b[4] - ld8 r18=[r2],-16 // r18=b5, r2=&tf_b[3] - mov b7=r16 - ;; -} -{ .mmi - ld8 r16=[r1],-16 // r16=b4, r1=&tf_b[2] - ld8 r19=[r2],-16 // r19=b3, r2=&tf_b[1] - mov b6=r17 - ;; -} -{ .mii - ld8 r17=[r1],-16 // r17=b2, r1=&tf_b[0] - mov b5=r18 - mov b4=r16 - ;; -} -{ .mii - ld8 r16=[r2],-16 // r16=b1, r2=&tf_ar_ec - mov b3=r19 - mov b2=r17 - ;; -} -{ .mmi - ld8 r17=[r1],-16 // r17=b0, r1=&tf_ar_lc - ld8 r18=[r2],-16 // r18=ar.ec, r2=&tf_ar_fptr - mov b1=r16 - ;; -} -{ .mmi - ld8 r16=[r1],-16 // r16=ar.lc, r1=&tf_ar_ccv - ld8 r19=[r2],-16 // r19=ar.fpsr, r1=&tf_ar_unat - mov b0=r17 - ;; -} -{ .mmi - ld8 r17=[r1],-16 // r17=ar.ccv, r1=&tf_ndirty - mov ar.fpsr=r19 - mov ar.ec=r18 - ;; -} -{ .mmi - ld8 r18=[r2],-16 // r18=ar.unat, r2=&tf_ar_rnat - mov ar.ccv=r17 - mov ar.lc=r16 - ;; -} -{ .mmb - ld8 r27=[r1],-16 // r1=&tf_ar_bspstore - ld8 r21=[r2],-16 // r2=&tf_cr_ifs - nop 28 - ;; -} -{ .mmi - mov ar.unat=r18 - ld8 r22=[r1],-16 // r1=&tf_ar_pfs - nop 29 -} -{ .mfb - ld8 r25=[r2],-16 // r2=&tf_ar_rsc - nop 30 -(p1) br.cond.dpnt.few 1f // don't switch bs if kernel - ;; -} -{ .mmi - alloc r16=ar.pfs,0,0,0,0 // discard current frame - ;; - nop 31 - shl r16=r27,16 // value for ar.rsc - ;; -} -{ .mmi - mov ar.rsc=r16 // setup for loadrs - ;; - loadrs // load user regs from kernel bs - nop 32 - ;; -} -{ .mmi - mov ar.bspstore=r22 - ;; - mov ar.rnat=r21 - nop 33 - ;; -} -1: -{ .mmb - ld8 r19=[r1],-16 // r1=&tf_pr - ld8 r20=[r2],-16 // r2=&tf_cr_ifa - nop 34 - ;; -} -{ .mmi - ld8 r27=[r1],-16 // r1=&tf_cr_isr - ld8 r28=[r2],-16 // r2=&tf_cr_ipsr - mov ar.pfs=r19 - ;; -} -{ .mmi - ld8 r29=[r1],-16 // r1=&tf_cr_iip - ld8 r30=[r2] - mov pr=r27,0x1ffff - ;; -} -{ .mmi - ld8 r31=[r1] - mov cr.ifs=r25 - mov r2=r23 - ;; -} -{ .mmi - mov cr.ifa=r28 - mov cr.iip=r31 - mov r1=r24 - ;; -} -{ .mmi - mov cr.ipsr=r30 - mov ar.rsc=r20 - nop 35 - ;; -} -{ .bbb - nop 36 - nop 37 - rfi - ;; -} -END(exception_restore) - - -/* - * exception_save: save interrupted state - * - * Arguments: - * b6 return address - * r16 saved b6 - * - * Return: - * r14 cr.iim value for break traps - * sp kernel stack pointer - * p1 true if user mode - * p2 true if kernel mode - * p3 true if interrupts were enabled - */ -ENTRY(exception_save, 0) -{ .mmi - rsm psr.dt // turn off data translations - ;; - srlz.d // serialize - mov r27=pr -} -{ .mmi - mov r30=cr.ipsr - ;; - mov r31=cr.iip - tbit.nz p3,p0=r30,14 // check for interrupt enable state -} -{ .mmi - mov r29=cr.isr - ;; - mov r26=sp // save sp - extr.u r17=r30,32,2 // extract ipsr.cpl - ;; -} -{ .mmi - cmp.eq p1,p2=r0,r17 // test for kernel mode - ;; -(p2) mov sp=ar.k6 // and switch to kernel stack - mov r24=r1 - ;; -} -{ .mii - mov r28=cr.ifa - add sp=-SIZEOF_TRAPFRAME,sp // reserve trapframe - ;; - dep r1=0,sp,61,3 // r1=&tf_flags - ;; -} -{ .mmi - st8 [r1]=r0,8 // zero flags, r1=&tf_cr_iip - ;; - mov r23=r2 - add r2=8,r1 // r2=&tf_cr_ipsr - ;; -} -{ .mmb - st8 [r1]=r31,16 // r1=&tf_cr_isr - st8 [r2]=r30,16 // r2=&tf_cr_ifa - nop 1 - ;; -} -{ .mmb - st8 [r1]=r29,16 // r1=&tf_pr - st8 [r2]=r28,16 // r2=&tf_ar_rsc - nop 2 - ;; -} -{ .mmi - st8 [r1]=r27,16 // r1=&tf_cr_pfs - mov r20=ar.rsc - mov r19=ar.pfs - ;; -} -{ .mmb - st8 [r2]=r20,16 // r2=&tf_cr_ifs - st8 [r1]=r19,16 // r1=&tf_ar_bspstore - cover - ;; -} -{ .mmi - mov ar.rsc=0 - ;; - mov r22=ar.bspstore - ;; -} -{ .mmi - mov r25=cr.ifs - mov r21=ar.rnat -(p1) mov r31=r22 // so we can figure out ndirty - ;; -} -{ .mmb -(p2) mov r31=ar.k5 // kernel backing store - st8 [r2]=r25,16 // r2=&tf_ar_rnat - nop 3 - ;; -} -{ .mmi - st8 [r1]=r22,16 // r1=&tf_ndirty -(p2) mov ar.bspstore=r31 // switch bspstore - nop 4 - ;; -} -{ .mmi - mov r17=ar.bsp - ;; - st8 [r2]=r21,16 // r2=&tf_ar_unat - sub r17=r17,r31 // ndirty (in bytes) - ;; -} -{ .mmi - st8 [r1]=r17,16 // r1=&tf_ar_ccv - mov ar.rsc=3 // switch RSE back on - mov r18=ar.lc -} -{ .mmi - mov r31=ar.unat - ;; - st8 [r2]=r31,16 // r2=&tf_ar_fpsr - mov r19=ar.ec -} -{ .mmi - mov r17=ar.ccv - ;; - st8 [r1]=r17,16 // r1=&tf_ar_lc - nop 5 - ;; -} -{ .mmi - mov r31=ar.fpsr - ;; - st8 [r2]=r31,16 // r2=&tf_ar_ec - mov r30=b0 - ;; -} -{ .mmi - st8 [r1]=r18,16 // r1=&tf_b[0] - ;; - st8 [r2]=r19,16 // r2=&tf_b[1] - mov r31=b1 -} -{ .mmi - st8 [r1]=r30,16 // r1=&tf_b[2] - ;; - st8 [r2]=r31,16 // r2=&tf_b[3] - mov r31=b2 - ;; -} -{ .mii - st8 [r1]=r31,16 // r1=&tf_b[4] - mov r17=b3 - ;; - mov r18=b4 -} -{ .mmi - st8 [r2]=r17,16 // r2=&tf_b[5] - ;; - st8 [r1]=r18,16 // r1=&tf_b[6] - mov r31=b5 - ;; -} -{ .mii - st8 [r2]=r31,16 // r2=&tf_b[7] - mov r18=b7 -} -{ .mmi - st8 [r1]=r16,16 // r1=&tf_r[FRAME_R1] - ;; - st8 [r2]=r18,16 // r2=&tf_r[FRAME_R2] - nop 7 - ;; -} -{ .mmb - .mem.offset 0,0 - st8.spill [r1]=r24,16 // r1=&tf_r[FRAME_R3] - .mem.offset 8,0 - st8.spill [r2]=r23,16 // r2=&tf_r[FRAME_R4] - nop 8 - ;; -} -{ .mmi - .mem.offset 16,0 - st8.spill [r1]=r3,16 // r1=&tf_r[FRAME_R5] - .mem.offset 32,0 - st8.spill [r2]=r4,16 // r2=&tf_r[FRAME_R6] - nop 9 - ;; -} -{ .mmb - .mem.offset 48,0 - st8.spill [r1]=r5,16 // r1=&tf_r[FRAME_R7] - .mem.offset 64,0 - st8.spill [r2]=r6,16 // r2=&tf_r[FRAME_R8] - nop 10 - ;; -} -{ .mmi - .mem.offset 80,0 - st8.spill [r1]=r7,16 // r1=&tf_r[FRAME_R9] - .mem.offset 96,0 - st8.spill [r2]=r8,16 // r2=&tf_r[FRAME_R10] - nop 11 - ;; -} -{ .mmb - .mem.offset 112,0 - st8.spill [r1]=r9,16 // r1=&tf_r[FRAME_R11] - .mem.offset 128,0 - st8.spill [r2]=r10,16 // r2=&tf_r[FRAME_SP] - nop 12 - ;; -} -{ .mmi - .mem.offset 144,0 - st8.spill [r1]=r11,16 // r1=&tf_r[FRAME_R13] - .mem.offset 160,0 - st8.spill [r2]=r26,16 // r2=&tf_r[FRAME_R14] - nop 13 - ;; -} -{ .mmb - .mem.offset 176,0 - st8.spill [r1]=r13,16 // r1=&tf_r[FRAME_R15] - .mem.offset 192,0 - st8.spill [r2]=r14,16 // r2=&tf_r[FRAME_R16] - nop 14 - ;; -} -{ .mfb - .mem.offset 208,0 - st8.spill [r1]=r15,16 // r1=&tf_r[FRAME_R17] - nop 15 - bsw.1 // switch to bank 1 - ;; -} -{ .mmi - .mem.offset 224,0 - st8.spill [r2]=r16,16 // r2=&tf_r[FRAME_R18] - .mem.offset 240,0 - st8.spill [r1]=r17,16 // r1=&tf_r[FRAME_R19] - nop 16 - ;; -} -{ .mmb - .mem.offset 256,0 - st8.spill [r2]=r18,16 // r2=&tf_r[FRAME_R20] - .mem.offset 272,0 - st8.spill [r1]=r19,16 // r1=&tf_r[FRAME_R21] - nop 17 - ;; -} -{ .mmi - .mem.offset 288,0 - st8.spill [r2]=r20,16 // r2=&tf_r[FRAME_R22] - .mem.offset 304,0 - st8.spill [r1]=r21,16 // r1=&tf_r[FRAME_R23] - nop 18 - ;; -} -{ .mmb - .mem.offset 320,0 - st8.spill [r2]=r22,16 // r2=&tf_r[FRAME_R24] - .mem.offset 336,0 - st8.spill [r1]=r23,16 // r1=&tf_r[FRAME_R25] - nop 19 - ;; -} -{ .mmi - .mem.offset 352,0 - st8.spill [r2]=r24,16 // r2=&tf_r[FRAME_R26] - .mem.offset 368,0 - st8.spill [r1]=r25,16 // r1=&tf_r[FRAME_R27] - nop 20 - ;; -} -{ .mmb - .mem.offset 384,0 - st8.spill [r2]=r26,16 // r2=&tf_r[FRAME_R28] - .mem.offset 400,0 - st8.spill [r1]=r27,16 // r1=&tf_r[FRAME_R29] - nop 21 - ;; -} -{ .mmi - .mem.offset 416,0 - st8.spill [r2]=r28,16 // r2=&tf_r[FRAME_R30] - .mem.offset 432,0 - st8.spill [r1]=r29,16 // r1=&tf_r[FRAME_R31] - nop 22 - ;; -} -{ .mmb - .mem.offset 448,0 - st8.spill [r2]=r30,16 // r2=&tf_f[FRAME_F6] - .mem.offset 464,0 - st8.spill [r1]=r31,24 // r1=&tf_f[FRAME_F7] - nop 23 - ;; -} -{ .mmi - stf.spill [r2]=f6,32 // r2=&tf_f[FRAME_F8] - stf.spill [r1]=f7,32 // r1=&tf_f[FRAME_F9] - nop 24 - ;; -} -{ .mmb - stf.spill [r2]=f8,32 // r2=&tf_f[FRAME_F10] - stf.spill [r1]=f9,32 // r1=&tf_f[FRAME_F11] - nop 25 - ;; -} -{ .mmi - stf.spill [r2]=f10,32 // r2=&tf_f[FRAME_F12] - stf.spill [r1]=f11,32 // r1=&tf_f[FRAME_F13] - nop 26 - ;; -} -{ .mmb - stf.spill [r2]=f12,32 // r2=&tf_f[FRAME_F14] - stf.spill [r1]=f13,32 // r1=&tf_f[FRAME_F15] - nop 27 - ;; -} -{ .mmi - stf.spill [r2]=f14 // - stf.spill [r1]=f15 // - nop 28 - ;; -} -{ .mlx - mov r14=cr.iim // break immediate - movl r1=__gp // kernel globals -} -{ .mmi - ssm psr.ic|psr.dt // enable interrupts & translation - ;; - srlz.i // serialize - nop 29 - ;; -} -{ .mfb - mov r13=ar.k4 // processor globals - nop 30 - br.sptk.few b6 // not br.ret - we were not br.call'ed - ;; -} -END(exception_save) - -/* - * System call entry point (via Break Instruction vector). - * - * Arguments: - * r15 System call number - * out0-out7 System call arguments - */ -ENTRY(do_syscall, 0) - .prologue - .unwabi @svr4, 'I' - .save rp,r0 - .body - // Save minimal state for syscall. - // We need to save enough state so that sendsig doesn't - // trash things if we take a signal during the system call. - // Essentially we need to save all the function-preserved - // state. Note that if we don't take a signal, we don't need - // to restore much of that state on the way out. Note also - // that when we save r4-r7 we spill their NaT bits into - // ar.unat. This register is preserved by the call to - // syscall() and if a full restore is needed, - // exception_restore will recover the NaT bits from ar.unat. - // The function-preserved state (including syscall number) is: - // - // r1,r4-r7,sp,r15 - // f16-f31 - // p1-p5,p16-p63 - // b0-b5 - // various ar's - // -{ .mmi // start reading high latency regs - mov r16=cr.ipsr // (13) - mov.m r17=ar.rsc // (13) - mov r18=sp // save user sp - ;; -} { .mmi - mov sp=ar.k6 // (13) kernel sp - mov r19=cr.isr // (13) - nop.i 0 -} { .mmi - mov.m ar.rsc=0 - ;; - mov.m r20=ar.bspstore // (13) - nop.i 0 -} { .mmi - mov.m r21=ar.k5 // (13) - mov.m r22=ar.rnat // (6) - nop.i 0 -} { .mmi - mov.m r23=ar.unat // (6) - rsm psr.dt // (5) physical addressing -} { .mii - mov r24=cr.iip // (2) - mov.i r25=ar.pfs // (2) - add sp=-SIZEOF_TRAPFRAME,sp // reserve trapframe - ;; -} { .mii - addl r27=FRAME_SYSCALL,r0 // (1) - mov r26=pr // (2) - dep r30=0,sp,61,3 // physical address -} { .mmi - srlz.d // serialize psr.dt - ;; - add r31=8,r30 // secondary pointer - ;; -} { .mmi - st8 [r30]=r27,16 // tf_flags - st8 [r31]=r24,16 // save cr.iip - mov r28=b0 - ;; -} { .mmi - st8 [r30]=r16,24 // save cr.ipsr, skip to pr - st8 [r31]=r19,24 // save cr.isr, skip to ar.rsc - mov r24=b1 - ;; -} { .mmi - st8 [r30]=r26,16 // save pr, skip to ar.pfs - st8 [r31]=r17,24 // save ar.rsc, skip to ar.bspstore - mov r27=b2 - ;; -} { .mii - st8 [r30]=r25,24 // save ar.pfs, skip to ar.rnat - mov r16=b3 - mov r17=b4 - ;; -} { .mmi - st8 [r31]=r20,24 // save ar.bspstore, skip to ar.unat - mov.m ar.bspstore=r21 // switch to kernel backing store - mov r29=b5 - ;; -} { .mmi - mov.m r20=ar.ccv - mov.m r21=ar.fpsr - nop.i 0 - ;; -} { .mmi - st8 [r30]=r22,24 // save ar.rnat, skip to ar.ccv - st8 [r31]=r23,16 // save ar.unat, skip to ar.fpsr - nop.i 0 - ;; -} { .mmi - st8 [r30]=r20,32 // save ar.ccv, skip to b0 - st8 [r31]=r21,32 // save ar.fpsr, skip to b1 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r28,16 // save b0, skip to b2 - st8 [r31]=r24,16 // save b1, skip to b3 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r27,16 // save b2, skip to b4 - st8 [r31]=r16,16 // save b3, skip to b5 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r17,TF_R_R1-(TF_B+4*8) // save b4, skip to r1 - st8 [r31]=r29,TF_R_R4-(TF_B+5*8) // save b5, skip to r4 - nop.i 0 - ;; -} { .mmi - st8 [r30]=r1,TF_R_R5-TF_R_R1 // save r1, skip to r5 - .mem.offset 8,0 - st8.spill [r31]=r4,16 // save r4, skip to r6 - nop.i 0 - ;; -} { .mmi - .mem.offset 0,0 - st8.spill [r30]=r5,16 // save r5, skip to r7 - .mem.offset 8,0 - st8.spill [r31]=r6,TF_R_SP-TF_R_R6 // save r6, skip to sp - nop.i 0 - ;; -} { .mmi - .mem.offset 0,0 - st8.spill [r30]=r7,TF_R_R15-TF_R_R7 // save r7, skip to r15 - st8 [r31]=r18 // save sp - nop.i 0 - ;; -} { .mmb - st8 [r30]=r15 // save r15 (syscall number) - add sp=-(8*8),sp // reserve stack for arguments - br.call.sptk.few b0=Lsaveargs // dump args -} { .mmb - mov.m r13=ar.k4 // processor globals - nop.m 0 - bsw.1 // switch back to bank 1 - ;; -} { .mmb - mov r16=sp // point at args - mov.m r17=ar.k5 // for calculating ndirty - cover // preserve user register frame - ;; -} { .mmi - mov r18=cr.ifs // record user's CFM - mov.m r19=ar.bsp // ndirty = ar.bsp - kbsp - add sp=-16,sp // reserve scratch space - ;; -} { .mmi - add r20=TF_CR_IFS+(8*8),r16 // point at cr.ifs - ssm psr.ic|psr.dt // reenable traps and translation - sub r19=r19,r17 // calculate ndirty - ;; -} { .mmi - srlz.i // serialize psr.ic and psr.dt - ;; - ssm psr.i // safe to take interrupts again - add r21=TF_NDIRTY+(8*8),r16 // point at ndirty - ;; -} { .mmi - st8 [r20]=r18 // save cr.ifs - st8 [r21]=r19 // save ndirty - ;; -} { .mmi - alloc r14=ar.pfs,0,1,3,0 - srlz.d // serialize psr.i - add loc0=(8*8),r16 // remember where trapframe is - ;; -} { .mlx - mov out0=r15 // syscall number (from user) - movl gp=__gp // kernel globals -} { .mmb - mov out1=r16 // arguments - add out2=(8*8),r16 // trapframe pointer - br.call.sptk.many rp=syscall // do the work -} { .mmi -3: rsm psr.i // we know that psr.i == 1 - add r14=PC_CURTHREAD,r13 // &curthread - nop.i 0 - ;; -} { .mmi - ld8 r14=[r14] // curthread - ;; - nop.i 0 - ;; -} { .mmi - ;; - add r14=TD_FLAGS,r14 // &curthread->td_flags - nop.i 0 - ;; -} { .mmi - ld4 r14=[r14] // curkse->ke_flags - ;; - nop.m 0 - tbit.nz p6,p7=r14,11 // TDF_ASTPENDING - ;; -} { .mib - nop.m 0 -(p7) tbit.nz.or.andcm p6,p7=r14,16 // TDF_NEEDRESCHED -(p7) br.cond.dptk 2f - ;; -} { .mmi - ssm psr.i // restore interrupts - ;; - srlz.d - mov out0=loc0 // trapframe argument to ast() -} { .mib - nop.m 0 - nop.i 0 - br.call.sptk.many rp=ast -} { .mib - nop.m 0 - nop.i 0 - br 3b -} { .mii -2: ld8 r14=[loc0] // check tf_flags - dep r15=0,loc0,61,3 // physical address of trapframe - ;; - tbit.z p6,p0=r14,0 // check FRAME_SYSCALL bit - ;; -} { .mib -(p6) add sp=-16,loc0 // do a full restore if clear - add r16=SIZEOF_TRAPFRAME,loc0 // new kernel sp -(p6) br.dpnt.many exception_restore -} { .mmi - rsm psr.dt|psr.ic|psr.i // get ready to restore - ;; - srlz.i // serialise psr.dt and psr.ic - add r30=TF_R_R15,r15 // point at r15 - ;; -} { .mmi - alloc r14=ar.pfs,0,0,0,0 // discard register frame - mov ar.k6=r16 // restore kernel sp - add r31=TF_R_SP,r15 // point at sp - ;; -} { .mmi - ld8 r15=[r30],TF_R_R10-TF_R_R15 // restore r15, skip to r10 - ld8 sp=[r31],TF_R_R9-TF_R_SP // restore user sp, skip to r9 - nop.i 0 - ;; -} { .mmi - ld8 r10=[r30],-16 // restore r10, skip to r8 - ld8 r9=[r31],TF_R_R1-TF_R_R9 // restore r9, skip to r1 - nop.i 0 - ;; -} { .mmi - ld8 r8=[r30],TF_B-TF_R_R8 // restore r8, skip to b0 - ld8 r1=[r31],TF_AR_FPSR-TF_R_R1 // restore r1, skip to ar.fpsr - nop.i 0 - ;; -} { .mmi - ld8 r16=[r30],-32 // restore b0, skip to ar.ccv - ld8 r17=[r31],-16 // restore ar.fpsr, skip to ar.unat - nop.i 0 - ;; -} { .mmi - ld8 r18=[r30],-16 // restore ar.ccv, skip to ndirty - ld8 r19=[r31],-16 // restore ar.unat, skip to ar.rnat - mov b0=r16 - ;; -} { .mmi - ld8 r20=[r30],-16 // restore ndirty, skip to ar.bspstore - ld8 r21=[r31],-16 // restore ar.rnat, skip to cr.ifs - nop.i 0 - ;; -} { .mmi - ld8 r16=[r30],-16 // restore ar.bspstore, skip to ar.pfs - mov ar.fpsr=r17 - shl r20=r20,16 // value for ar.rsc - ;; -} { .mmi - ld8 r22=[r31],-16 // restore cr.ifs, skip to ar.rsc - mov.m ar.ccv=r18 - nop.i 0 - ;; -} { .mmi - ld8 r17=[r30],-16 // restore ar.pfs, skip to pr - mov.m ar.unat=r19 - nop.i 0 - ;; -} { .mmi - ld8 r18=[r31],-32 // restore ar.rsc, skip to cr.ipsr - mov.m ar.rsc=r20 // setup for loadrs - nop.i 0 - ;; -} { .mmi - loadrs // restore user stacked registers - ;; - mov.m ar.bspstore=r16 // back to user backing store - mov.i ar.pfs=r17 - ;; -} { .mmi - mov.m ar.rnat=r21 - mov.m ar.rsc=r18 - nop.i 0 - ;; -} { .mmi - ld8 r16=[r30],-32 // restore pr, skip to cr.iip - ld8 r17=[r31] // restore cr.ipsr - nop.i 0 - ;; -} { .mmi - ld8 r18=[r30] // restore cr.iip - mov cr.ifs=r22 - nop.i 0 - ;; -} { .mmi - mov cr.iip=r18 - mov cr.ipsr=r17 - mov pr=r16,0x1ffff - ;; -} { .bbb - rfi -} - - // This is done as a function call to make sure that we only - // have output registers in the register frame. It also gives - // us a chance to use alloc to round up to 8 arguments for - // simplicity. - // - // We are still running in physical mode with psr.ic==0 because - // we haven't yet covered the user's register frame to get a - // value for cr.ifs -Lsaveargs: -{ .mii - alloc r14=ar.pfs,0,0,8,0 // round up to 8 outputs - extr.u r31=sp,0,61 // physical address - ;; - add r30=8,r31 - ;; -} { .mmi - st8 [r31]=r32,16 - st8 [r30]=r33,16 - ;; -} { .mmi - st8 [r31]=r34,16 - st8 [r30]=r35,16 - ;; -} { .mmi - st8 [r31]=r36,16 - st8 [r30]=r37,16 - ;; -} { .mmb - st8 [r31]=r38 - st8 [r30]=r39 - br.ret.sptk.many b0 -} - .global do_syscall_end -do_syscall_end: - -END(do_syscall) diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c index 158b388..302c441 100644 --- a/sys/ia64/ia64/genassym.c +++ b/sys/ia64/ia64/genassym.c @@ -37,8 +37,6 @@ * $FreeBSD$ */ -#include "opt_ia32.h" - #include <sys/param.h> #include <sys/systm.h> #include <sys/assym.h> @@ -65,87 +63,57 @@ #include <net/if.h> #include <netinet/in.h> -#ifdef IA32 -ASSYM(IA32, IA32); -#endif - -ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); -ASSYM(PC_IDLETHREAD, offsetof(struct pcpu, pc_idlethread)); -ASSYM(PC_FPCURTHREAD, offsetof(struct pcpu, pc_fpcurthread)); -ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); -ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); -ASSYM(PC_CURRENT_PMAP, offsetof(struct pcpu, pc_current_pmap)); - -ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); -ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_UNOWNED, MTX_UNOWNED); +ASSYM(DT_NULL, DT_NULL); +ASSYM(DT_RELA, DT_RELA); +ASSYM(DT_RELAENT, DT_RELAENT); +ASSYM(DT_RELASZ, DT_RELASZ); +ASSYM(DT_SYMTAB, DT_SYMTAB); +ASSYM(DT_SYMENT, DT_SYMENT); -ASSYM(TD_PROC, offsetof(struct thread, td_proc)); -ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); -ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack)); -ASSYM(TD_MD_FLAGS, offsetof(struct thread, td_md.md_flags)); +ASSYM(EFAULT, EFAULT); +ASSYM(ENAMETOOLONG, ENAMETOOLONG); +ASSYM(ERESTART, ERESTART); -ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); +ASSYM(FRAME_SYSCALL, FRAME_SYSCALL); -ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); -ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); +ASSYM(KSTACK_PAGES, KSTACK_PAGES); -ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS); +ASSYM(MC_PRESERVED, offsetof(mcontext_t, mc_preserved)); +ASSYM(MC_PRESERVED_FP, offsetof(mcontext_t, mc_preserved_fp)); +ASSYM(MC_SPECIAL, offsetof(mcontext_t, mc_special)); +ASSYM(MC_SPECIAL_BSPSTORE, offsetof(mcontext_t, mc_special.bspstore)); +ASSYM(MC_SPECIAL_RNAT, offsetof(mcontext_t, mc_special.rnat)); -ASSYM(FRAME_SYSCALL, FRAME_SYSCALL); +ASSYM(PAGE_SHIFT, PAGE_SHIFT); +ASSYM(PAGE_SIZE, PAGE_SIZE); -ASSYM(TF_CR_IPSR, offsetof(struct trapframe, tf_cr_ipsr)); -ASSYM(TF_CR_IFS, offsetof(struct trapframe, tf_cr_ifs)); -ASSYM(TF_NDIRTY, offsetof(struct trapframe, tf_ndirty)); -ASSYM(TF_AR_FPSR, offsetof(struct trapframe, tf_ar_fpsr)); -ASSYM(TF_B, offsetof(struct trapframe, tf_b)); -ASSYM(TF_R, offsetof(struct trapframe, tf_r)); -ASSYM(TF_R_R1, offsetof(struct trapframe, tf_r[FRAME_R1])); -ASSYM(TF_R_R2, offsetof(struct trapframe, tf_r[FRAME_R2])); -ASSYM(TF_R_R3, offsetof(struct trapframe, tf_r[FRAME_R3])); -ASSYM(TF_R_R4, offsetof(struct trapframe, tf_r[FRAME_R4])); -ASSYM(TF_R_R5, offsetof(struct trapframe, tf_r[FRAME_R5])); -ASSYM(TF_R_R6, offsetof(struct trapframe, tf_r[FRAME_R6])); -ASSYM(TF_R_R7, offsetof(struct trapframe, tf_r[FRAME_R7])); -ASSYM(TF_R_R8, offsetof(struct trapframe, tf_r[FRAME_R8])); -ASSYM(TF_R_R9, offsetof(struct trapframe, tf_r[FRAME_R9])); -ASSYM(TF_R_R10, offsetof(struct trapframe, tf_r[FRAME_R10])); -ASSYM(TF_R_R11, offsetof(struct trapframe, tf_r[FRAME_R11])); -ASSYM(TF_R_SP, offsetof(struct trapframe, tf_r[FRAME_SP])); -ASSYM(TF_R_R13, offsetof(struct trapframe, tf_r[FRAME_R13])); -ASSYM(TF_R_R14, offsetof(struct trapframe, tf_r[FRAME_R14])); -ASSYM(TF_R_R15, offsetof(struct trapframe, tf_r[FRAME_R15])); -ASSYM(TF_F, offsetof(struct trapframe, tf_f)); +ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); +ASSYM(PC_CURRENT_PMAP, offsetof(struct pcpu, pc_current_pmap)); +ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); +ASSYM(PC_IDLETHREAD, offsetof(struct pcpu, pc_idlethread)); ASSYM(PCB_CURRENT_PMAP, offsetof(struct pcb, pcb_current_pmap)); ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); -ASSYM(PCB_RP, offsetof(struct pcb, pcb_rp)); -ASSYM(PCB_UNAT47, offsetof(struct pcb, pcb_unat47)); +ASSYM(PCB_SPECIAL_RP, offsetof(struct pcb, pcb_special.rp)); -ASSYM(UC_MCONTEXT_MC_AR_BSP, offsetof(ucontext_t, uc_mcontext.mc_ar_bsp)); -ASSYM(UC_MCONTEXT_MC_AR_RNAT, offsetof(ucontext_t, uc_mcontext.mc_ar_rnat)); +ASSYM(R_IA64_DIR64LSB, R_IA64_DIR64LSB); +ASSYM(R_IA64_FPTR64LSB, R_IA64_FPTR64LSB); +ASSYM(R_IA64_NONE, R_IA64_NONE); +ASSYM(R_IA64_REL64LSB, R_IA64_REL64LSB); -ASSYM(EFAULT, EFAULT); -ASSYM(ENAMETOOLONG, ENAMETOOLONG); +ASSYM(SIZEOF_PCB, sizeof(struct pcb)); +ASSYM(SIZEOF_SPECIAL, sizeof(struct _special)); +ASSYM(SIZEOF_TRAPFRAME, sizeof(struct trapframe)); -ASSYM(PAGE_SHIFT, PAGE_SHIFT); -ASSYM(PAGE_SIZE, PAGE_SIZE); -ASSYM(KSTACK_PAGES, KSTACK_PAGES); +ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); +ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack)); +ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); -ASSYM(SIZEOF_TRAPFRAME, sizeof(struct trapframe)); -ASSYM(SIZEOF_PCB, sizeof(struct pcb)); +ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); +ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); -ASSYM(DT_NULL, DT_NULL); -ASSYM(DT_RELA, DT_RELA); -ASSYM(DT_RELASZ, DT_RELASZ); -ASSYM(DT_SYMTAB, DT_SYMTAB); -ASSYM(DT_SYMENT, DT_SYMENT); -ASSYM(DT_RELAENT, DT_RELAENT); -ASSYM(R_IA64_NONE, R_IA64_NONE); -ASSYM(R_IA64_DIR64LSB, R_IA64_DIR64LSB); -ASSYM(R_IA64_FPTR64LSB, R_IA64_FPTR64LSB); -ASSYM(R_IA64_REL64LSB, R_IA64_REL64LSB); +ASSYM(TF_SPECIAL_NDIRTY, offsetof(struct trapframe, tf_special.ndirty)); + +ASSYM(UC_MCONTEXT, offsetof(ucontext_t, uc_mcontext)); -ASSYM(PAL_PTCE_INFO, PAL_PTCE_INFO); -ASSYM(PAL_FREQ_RATIOS, PAL_FREQ_RATIOS); -ASSYM(PAL_VM_SUMMARY, PAL_VM_SUMMARY); +ASSYM(VM_MAX_ADDRESS, VM_MAX_ADDRESS); diff --git a/sys/ia64/ia64/interrupt.c b/sys/ia64/ia64/interrupt.c index bc2d324..a9539a9 100644 --- a/sys/ia64/ia64/interrupt.c +++ b/sys/ia64/ia64/interrupt.c @@ -152,6 +152,9 @@ interrupt(u_int64_t vector, struct trapframe *framep) } else if (vector == ipi_vector[IPI_AST]) { asts[PCPU_GET(cpuid)]++; CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); + } else if (vector == ipi_vector[IPI_HIGH_FP]) { + if (PCPU_GET(fpcurthread) != NULL) + ia64_highfp_save(PCPU_GET(fpcurthread)); } else if (vector == ipi_vector[IPI_RENDEZVOUS]) { rdvs[PCPU_GET(cpuid)]++; CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); diff --git a/sys/ia64/ia64/locore.S b/sys/ia64/ia64/locore.S index 61671b6..69550d1 100644 --- a/sys/ia64/ia64/locore.S +++ b/sys/ia64/ia64/locore.S @@ -25,32 +25,6 @@ * * $FreeBSD$ */ -/* - * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University. - * All rights reserved. - * - * Author: Chris G. Demetriou - * - * Permission to use, copy, modify and distribute this software and - * its documentation is hereby granted, provided that both the copyright - * notice and this permission notice appear in all copies of the - * software, derivative works or modified versions, and any portions - * thereof, and that both notices appear in supporting documentation. - * - * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" - * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND - * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * - * Carnegie Mellon requests users of this software to return to - * - * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU - * School of Computer Science - * Carnegie Mellon University - * Pittsburgh PA 15213-3890 - * - * any improvements or extensions that they make and grant Carnegie the - * rights to redistribute these changes. - */ #include <machine/asm.h> #include <machine/ia64_cpu.h> @@ -77,22 +51,33 @@ kstack: .space KSTACK_PAGES * PAGE_SIZE * register r8. */ ENTRY(__start, 1) +{ .mlx mov ar.rsc=0 movl r16=ia64_vector_table // set up IVT early ;; +} +{ .mlx mov cr.iva=r16 movl r16=kstack ;; +} +{ .mmi srlz.i ;; + ssm IA64_PSR_DFH mov r17=KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16 ;; +} +{ .mlx add sp=r16,r17 // proc0's stack movl gp=__gp // find kernel globals ;; +} +{ .mlx mov ar.bspstore=r16 // switch backing store movl r16=pa_bootinfo ;; +} st8 [r16]=r8 // save the PA of the bootinfo block loadrs // invalidate regs ;; @@ -117,7 +102,7 @@ ENTRY(__start, 1) ;; ld8 out0=[out0] ;; - add r16=PCB_RP,out0 // return to mi_startup_trampoline + add r16=PCB_SPECIAL_RP,out0 // return to mi_startup_trampoline movl r17=mi_startup_trampoline ;; st8 [r16]=r17 @@ -133,7 +118,7 @@ ENTRY(mi_startup_trampoline, 0) .prologue .save rp,r0 .body - + br.call.sptk.many rp=mi_startup // Should never happen @@ -141,6 +126,59 @@ ENTRY(mi_startup_trampoline, 0) END(mi_startup_trampoline) +/* + * fork_trampoline() + * + * Arrange for a function to be invoked neatly, after a cpu_switch(). + * + * Invokes fork_exit() passing in three arguments: a callout function, an + * argument to the callout, and a trapframe pointer. For child processes + * returning from fork(2), the argument is a pointer to the child process. + * + * The callout function and its argument is in the trapframe in scratch + * registers r2 and r3. + */ +ENTRY(fork_trampoline, 0) + .prologue + .save rp,r0 + .body +{ .mmi + alloc r14=ar.pfs,0,0,3,0 + add r15=32+SIZEOF_SPECIAL+8,sp + add r16=32+SIZEOF_SPECIAL+16,sp + ;; +} +{ .mmi + ld8 out0=[r15] + ld8 out1=[r16] + nop 0 +} +{ .mfb + add out2=16,sp + nop 0 + br.call.sptk rp=fork_exit + ;; +} + // If we get back here, it means we're a user space process that's + // the immediate result of fork(2). + .global enter_userland + .type enter_userland, @function +enter_userland: +{ .mmi + add r14=24,sp + ;; + ld8 r14=[r14] + nop 0 + ;; +} +{ .mbb + cmp.eq p6,p7=r0,r14 +(p6) br.sptk exception_restore +(p7) br.sptk epc_syscall_return + ;; +} +END(fork_trampoline) + #ifdef SMP /* * AP wake-up entry point. The handoff state is similar as for the BSP, @@ -198,7 +236,7 @@ ENTRY(os_boot_rendez,0) ;; 1: mov r16 = ip add r17 = 2f-1b, r17 - movl r18 = (IA64_PSR_AC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT|IA64_PSR_BN) + movl r18 = (IA64_PSR_AC|IA64_PSR_BN|IA64_PSR_DFH|IA64_PSR_DT|IA64_PSR_IC|IA64_PSR_IT|IA64_PSR_RT) ;; add r17 = r17, r16 mov cr.ipsr = r18 @@ -209,142 +247,54 @@ ENTRY(os_boot_rendez,0) rfi .align 32 -2: movl r16 = ia64_vector_table // set up IVT early +2: +{ .mlx + mov ar.rsc = 0 + movl r16 = ia64_vector_table // set up IVT early ;; +} +{ .mlx mov cr.iva = r16 + movl r16 = ap_stack ;; +} +{ .mmi srlz.i ;; - movl r16 = ap_stack - movl r17 = ap_pcpu - mov ar.rsc = 0 - movl gp = __gp - ;; ld8 r16 = [r16] - ld8 r17 = [r17] mov r18 = KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16 ;; - add sp = r18, r16 +} +{ .mlx mov ar.bspstore = r16 - mov ar.k4 = r17 - mov r13 = r17 /* gas doesn't know tp as an alias for r13 */ + movl gp = __gp ;; +} +{ .mmi loadrs - movl r16 = ia64_pal_base - ;; - mov ar.rsc = 3 - ld8 r16 = [r16] - ;; - cmp.eq p1, p0 = 0, r16 -(p1) br.cond.spnt 1f - ;; - mov r18 = 28<<2 - movl r17 = 7<<61 - ;; - mov cr.itir = r18 - or r17 = r17, r16 - mov r16 = (PTE_P|PTE_MA_WB|PTE_A|PTE_D|PTE_PL_KERN|PTE_AR_RWX) - ;; - mov cr.ifa = r17 - extr.u r18 = r17, 12, 38 - ;; - srlz.i - shl r18 = r18, 12 - ;; - add r17 = 1, r0 - or r16 = r16, r18 - ;; - itr.i itr[r17] = r16 ;; - srlz.i - ;; -1: alloc r16 = ar.pfs, 0, 0, 0, 0 + alloc r17 = ar.pfs, 0, 0, 0, 0 + add sp = r18, r16 ;; +} +{ .mfb + mov ar.rsc = 3 + nop 0 br.call.sptk.few rp = ia64_ap_startup + ;; +} /* NOT REACHED */ -9: br 9b +9: +{ .mfb + nop 0 + nop 0 + br.sptk 9b + ;; +} END(os_boot_rendez) #endif /* !SMP */ -/**************************************************************************/ - -/* - * Signal "trampoline" code. Invoked from RTE setup by sendsig(). - * - * On entry, registers look like: - * - * r14 signal number - * r15 pointer to siginfo_t - * r16 pointer to signal context frame (scp) - * r17 address of handler function descriptor - * r18 address of new backing store (if any) - * sp+16 pointer to sigframe - */ - -ENTRY(sigcode,0) - ld8 r8=[r17],8 // function address - ;; - ld8 gp=[r17] // function's gp value - mov b6=r8 // transfer to a branch register - cover - ;; - add r8=UC_MCONTEXT_MC_AR_BSP,r16 // address or mc_ar_bsp - mov r9=ar.bsp // save ar.bsp - ;; - st8 [r8]=r9 - cmp.eq p1,p2=r0,r18 // check for new bs -(p1) br.cond.sptk.few 1f // branch if not switching - flushrs // flush out to old bs - mov ar.rsc=0 // switch off RSE - add r8=UC_MCONTEXT_MC_AR_RNAT,r16 // address of mc_ar_rnat - ;; - mov r9=ar.rnat // value of ar.rnat after flush - mov ar.bspstore=r18 // point at new bs - ;; - st8 [r8]=r9 // remember ar.rnat - mov ar.rsc=15 // XXX bogus value - check - invala - ;; -1: alloc r5=ar.pfs,0,0,3,0 // register frame for call - ;; - mov out0=r14 // signal number - mov out1=r15 // siginfo - mov out2=r16 // ucontext - mov r4=r16 // save from call - br.call.sptk.few rp=b6 // call the signal handler - ;; - alloc r14=ar.pfs,0,0,0,0 // discard call frame - ;; - flushrs - ;; -(p1) br.cond.sptk.few 2f // note: p1 is preserved - mov ar.rsc=0 - add r8=UC_MCONTEXT_MC_AR_RNAT,r4 // address of mc_ar_rnat - ;; - ld8 r9=[r8] - ;; - add r8=UC_MCONTEXT_MC_AR_BSP,r4 // address of mc_ar_bsp - ;; - ld8 r10=[r8] - ;; - mov ar.bspstore=r10 - ;; - mov ar.rnat=r9 - mov ar.rsc=15 - ;; -2: CALLSYS_NOERROR(sigreturn) // call sigreturn() - alloc r14=ar.pfs,0,0,1,0 ;; - mov out0=ret0 // if that failed, get error code - CALLSYS_NOERROR(exit) // and call exit() with it. -XENTRY(esigcode) - END(sigcode) - - .data - EXPORT(szsigcode) - .quad esigcode-sigcode - .text - /* * Create a default interrupt name table. The first entry (vector 0) is * hardwaired to the clock interrupt. diff --git a/sys/ia64/ia64/locore.s b/sys/ia64/ia64/locore.s index 61671b6..69550d1 100644 --- a/sys/ia64/ia64/locore.s +++ b/sys/ia64/ia64/locore.s @@ -25,32 +25,6 @@ * * $FreeBSD$ */ -/* - * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University. - * All rights reserved. - * - * Author: Chris G. Demetriou - * - * Permission to use, copy, modify and distribute this software and - * its documentation is hereby granted, provided that both the copyright - * notice and this permission notice appear in all copies of the - * software, derivative works or modified versions, and any portions - * thereof, and that both notices appear in supporting documentation. - * - * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" - * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND - * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * - * Carnegie Mellon requests users of this software to return to - * - * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU - * School of Computer Science - * Carnegie Mellon University - * Pittsburgh PA 15213-3890 - * - * any improvements or extensions that they make and grant Carnegie the - * rights to redistribute these changes. - */ #include <machine/asm.h> #include <machine/ia64_cpu.h> @@ -77,22 +51,33 @@ kstack: .space KSTACK_PAGES * PAGE_SIZE * register r8. */ ENTRY(__start, 1) +{ .mlx mov ar.rsc=0 movl r16=ia64_vector_table // set up IVT early ;; +} +{ .mlx mov cr.iva=r16 movl r16=kstack ;; +} +{ .mmi srlz.i ;; + ssm IA64_PSR_DFH mov r17=KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16 ;; +} +{ .mlx add sp=r16,r17 // proc0's stack movl gp=__gp // find kernel globals ;; +} +{ .mlx mov ar.bspstore=r16 // switch backing store movl r16=pa_bootinfo ;; +} st8 [r16]=r8 // save the PA of the bootinfo block loadrs // invalidate regs ;; @@ -117,7 +102,7 @@ ENTRY(__start, 1) ;; ld8 out0=[out0] ;; - add r16=PCB_RP,out0 // return to mi_startup_trampoline + add r16=PCB_SPECIAL_RP,out0 // return to mi_startup_trampoline movl r17=mi_startup_trampoline ;; st8 [r16]=r17 @@ -133,7 +118,7 @@ ENTRY(mi_startup_trampoline, 0) .prologue .save rp,r0 .body - + br.call.sptk.many rp=mi_startup // Should never happen @@ -141,6 +126,59 @@ ENTRY(mi_startup_trampoline, 0) END(mi_startup_trampoline) +/* + * fork_trampoline() + * + * Arrange for a function to be invoked neatly, after a cpu_switch(). + * + * Invokes fork_exit() passing in three arguments: a callout function, an + * argument to the callout, and a trapframe pointer. For child processes + * returning from fork(2), the argument is a pointer to the child process. + * + * The callout function and its argument is in the trapframe in scratch + * registers r2 and r3. + */ +ENTRY(fork_trampoline, 0) + .prologue + .save rp,r0 + .body +{ .mmi + alloc r14=ar.pfs,0,0,3,0 + add r15=32+SIZEOF_SPECIAL+8,sp + add r16=32+SIZEOF_SPECIAL+16,sp + ;; +} +{ .mmi + ld8 out0=[r15] + ld8 out1=[r16] + nop 0 +} +{ .mfb + add out2=16,sp + nop 0 + br.call.sptk rp=fork_exit + ;; +} + // If we get back here, it means we're a user space process that's + // the immediate result of fork(2). + .global enter_userland + .type enter_userland, @function +enter_userland: +{ .mmi + add r14=24,sp + ;; + ld8 r14=[r14] + nop 0 + ;; +} +{ .mbb + cmp.eq p6,p7=r0,r14 +(p6) br.sptk exception_restore +(p7) br.sptk epc_syscall_return + ;; +} +END(fork_trampoline) + #ifdef SMP /* * AP wake-up entry point. The handoff state is similar as for the BSP, @@ -198,7 +236,7 @@ ENTRY(os_boot_rendez,0) ;; 1: mov r16 = ip add r17 = 2f-1b, r17 - movl r18 = (IA64_PSR_AC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT|IA64_PSR_BN) + movl r18 = (IA64_PSR_AC|IA64_PSR_BN|IA64_PSR_DFH|IA64_PSR_DT|IA64_PSR_IC|IA64_PSR_IT|IA64_PSR_RT) ;; add r17 = r17, r16 mov cr.ipsr = r18 @@ -209,142 +247,54 @@ ENTRY(os_boot_rendez,0) rfi .align 32 -2: movl r16 = ia64_vector_table // set up IVT early +2: +{ .mlx + mov ar.rsc = 0 + movl r16 = ia64_vector_table // set up IVT early ;; +} +{ .mlx mov cr.iva = r16 + movl r16 = ap_stack ;; +} +{ .mmi srlz.i ;; - movl r16 = ap_stack - movl r17 = ap_pcpu - mov ar.rsc = 0 - movl gp = __gp - ;; ld8 r16 = [r16] - ld8 r17 = [r17] mov r18 = KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16 ;; - add sp = r18, r16 +} +{ .mlx mov ar.bspstore = r16 - mov ar.k4 = r17 - mov r13 = r17 /* gas doesn't know tp as an alias for r13 */ + movl gp = __gp ;; +} +{ .mmi loadrs - movl r16 = ia64_pal_base - ;; - mov ar.rsc = 3 - ld8 r16 = [r16] - ;; - cmp.eq p1, p0 = 0, r16 -(p1) br.cond.spnt 1f - ;; - mov r18 = 28<<2 - movl r17 = 7<<61 - ;; - mov cr.itir = r18 - or r17 = r17, r16 - mov r16 = (PTE_P|PTE_MA_WB|PTE_A|PTE_D|PTE_PL_KERN|PTE_AR_RWX) - ;; - mov cr.ifa = r17 - extr.u r18 = r17, 12, 38 - ;; - srlz.i - shl r18 = r18, 12 - ;; - add r17 = 1, r0 - or r16 = r16, r18 - ;; - itr.i itr[r17] = r16 ;; - srlz.i - ;; -1: alloc r16 = ar.pfs, 0, 0, 0, 0 + alloc r17 = ar.pfs, 0, 0, 0, 0 + add sp = r18, r16 ;; +} +{ .mfb + mov ar.rsc = 3 + nop 0 br.call.sptk.few rp = ia64_ap_startup + ;; +} /* NOT REACHED */ -9: br 9b +9: +{ .mfb + nop 0 + nop 0 + br.sptk 9b + ;; +} END(os_boot_rendez) #endif /* !SMP */ -/**************************************************************************/ - -/* - * Signal "trampoline" code. Invoked from RTE setup by sendsig(). - * - * On entry, registers look like: - * - * r14 signal number - * r15 pointer to siginfo_t - * r16 pointer to signal context frame (scp) - * r17 address of handler function descriptor - * r18 address of new backing store (if any) - * sp+16 pointer to sigframe - */ - -ENTRY(sigcode,0) - ld8 r8=[r17],8 // function address - ;; - ld8 gp=[r17] // function's gp value - mov b6=r8 // transfer to a branch register - cover - ;; - add r8=UC_MCONTEXT_MC_AR_BSP,r16 // address or mc_ar_bsp - mov r9=ar.bsp // save ar.bsp - ;; - st8 [r8]=r9 - cmp.eq p1,p2=r0,r18 // check for new bs -(p1) br.cond.sptk.few 1f // branch if not switching - flushrs // flush out to old bs - mov ar.rsc=0 // switch off RSE - add r8=UC_MCONTEXT_MC_AR_RNAT,r16 // address of mc_ar_rnat - ;; - mov r9=ar.rnat // value of ar.rnat after flush - mov ar.bspstore=r18 // point at new bs - ;; - st8 [r8]=r9 // remember ar.rnat - mov ar.rsc=15 // XXX bogus value - check - invala - ;; -1: alloc r5=ar.pfs,0,0,3,0 // register frame for call - ;; - mov out0=r14 // signal number - mov out1=r15 // siginfo - mov out2=r16 // ucontext - mov r4=r16 // save from call - br.call.sptk.few rp=b6 // call the signal handler - ;; - alloc r14=ar.pfs,0,0,0,0 // discard call frame - ;; - flushrs - ;; -(p1) br.cond.sptk.few 2f // note: p1 is preserved - mov ar.rsc=0 - add r8=UC_MCONTEXT_MC_AR_RNAT,r4 // address of mc_ar_rnat - ;; - ld8 r9=[r8] - ;; - add r8=UC_MCONTEXT_MC_AR_BSP,r4 // address of mc_ar_bsp - ;; - ld8 r10=[r8] - ;; - mov ar.bspstore=r10 - ;; - mov ar.rnat=r9 - mov ar.rsc=15 - ;; -2: CALLSYS_NOERROR(sigreturn) // call sigreturn() - alloc r14=ar.pfs,0,0,1,0 ;; - mov out0=ret0 // if that failed, get error code - CALLSYS_NOERROR(exit) // and call exit() with it. -XENTRY(esigcode) - END(sigcode) - - .data - EXPORT(szsigcode) - .quad esigcode-sigcode - .text - /* * Create a default interrupt name table. The first entry (vector 0) is * hardwaired to the clock interrupt. diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index dd3fc59..18d2083 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -73,6 +73,9 @@ #include <machine/mca.h> #include <machine/pal.h> #include <machine/sal.h> +#ifdef SMP +#include <machine/smp.h> +#endif #include <machine/bootinfo.h> #include <machine/mutex.h> #include <machine/vmparam.h> @@ -83,7 +86,6 @@ #include <machine/sigframe.h> #include <machine/efi.h> #include <machine/inst.h> -#include <machine/rse.h> #include <machine/unwind.h> #include <i386/include/specialreg.h> @@ -102,6 +104,10 @@ vm_offset_t proc0kstack; extern u_int64_t kernel_text[], _end[]; +extern u_int64_t ia64_gateway_page[]; +extern u_int64_t break_sigtramp[]; +extern u_int64_t epc_sigtramp[]; + FPSWA_INTERFACE *fpswa_interface; u_int64_t ia64_pal_base; @@ -206,6 +212,44 @@ cpu_startup(dummy) } void +cpu_switch(struct thread *old, struct thread *new) +{ + struct pcb *oldpcb, *newpcb; + + oldpcb = old->td_pcb; + oldpcb->pcb_current_pmap = PCPU_GET(current_pmap); +#if IA32 + ia32_savectx(oldpcb); +#endif + if (!savectx(oldpcb)) { + newpcb = new->td_pcb; + pmap_install(newpcb->pcb_current_pmap); + PCPU_SET(curthread, new); +#if IA32 + ia32_restorectx(newpcb); +#endif + restorectx(newpcb); + } +} + +void +cpu_throw(struct thread *old __unused, struct thread *new) +{ + struct pcb *newpcb; + + newpcb = new->td_pcb; + pmap_install(newpcb->pcb_current_pmap); + PCPU_SET(curthread, new); +#if IA32 + ia32_restorectx(newpcb); +#endif + restorectx(newpcb); + /* We should not get here. */ + panic("cpu_throw: restorectx() returned"); + /* NOTREACHED */ +} + +void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { KASSERT(size >= sizeof(struct pcpu) + sizeof(struct pcb), @@ -278,16 +322,20 @@ map_pal_code(void) pte.pte_ar = PTE_AR_RWX; pte.pte_ppn = ia64_pal_base >> 12; - __asm __volatile("mov %0=psr;;" : "=r" (psr)); - __asm __volatile("rsm psr.ic|psr.i;; srlz.i;;"); - __asm __volatile("mov cr.ifa=%0" :: + __asm __volatile("ptr.d %0,%1; ptr.i %0,%1" :: + "r"(IA64_PHYS_TO_RR7(ia64_pal_base)), "r"(28 << 2)); + + __asm __volatile("mov %0=psr" : "=r"(psr)); + __asm __volatile("rsm psr.ic|psr.i"); + __asm __volatile("srlz.i"); + __asm __volatile("mov cr.ifa=%0" :: "r"(IA64_PHYS_TO_RR7(ia64_pal_base))); - __asm __volatile("mov cr.itir=%0" :: "r"(28 << 2)); - __asm __volatile("srlz.i;;"); - __asm __volatile("itr.i itr[%0]=%1;;" :: - "r"(2), "r"(*(u_int64_t*)&pte)); - __asm __volatile("srlz.i;;"); - __asm __volatile("mov psr.l=%0;; srlz.i;;" :: "r" (psr)); + __asm __volatile("mov cr.itir=%0" :: "r"(28 << 2)); + __asm __volatile("itr.d dtr[%0]=%1" :: "r"(1), "r"(*(u_int64_t*)&pte)); + __asm __volatile("srlz.d"); /* XXX not needed. */ + __asm __volatile("itr.i itr[%0]=%1" :: "r"(1), "r"(*(u_int64_t*)&pte)); + __asm __volatile("mov psr.l=%0" :: "r" (psr)); + __asm __volatile("srlz.i"); } void @@ -306,20 +354,53 @@ map_port_space(void) pte.pte_a = 1; pte.pte_d = 1; pte.pte_pl = PTE_PL_KERN; - pte.pte_ar = PTE_AR_RWX; + pte.pte_ar = PTE_AR_RW; pte.pte_ppn = ia64_port_base >> 12; - __asm __volatile("mov %0=psr;;" : "=r" (psr)); - __asm __volatile("rsm psr.ic|psr.i;; srlz.i;;"); - __asm __volatile("mov cr.ifa=%0" :: - "r"(IA64_PHYS_TO_RR6(ia64_port_base))); + __asm __volatile("ptr.d %0,%1" :: "r"(ia64_port_base), "r"(24 << 2)); + + __asm __volatile("mov %0=psr" : "=r" (psr)); + __asm __volatile("rsm psr.ic|psr.i"); + __asm __volatile("srlz.d"); + __asm __volatile("mov cr.ifa=%0" :: "r"(ia64_port_base)); /* XXX We should use the size from the memory descriptor. */ - __asm __volatile("mov cr.itir=%0" :: "r"(24 << 2)); - __asm __volatile("srlz.i;;"); - __asm __volatile("itr.i itr[%0]=%1;;" :: - "r"(1), "r"(*(u_int64_t*)&pte)); - __asm __volatile("srlz.i;;"); - __asm __volatile("mov psr.l=%0;; srlz.i;;" :: "r" (psr)); + __asm __volatile("mov cr.itir=%0" :: "r"(24 << 2)); + __asm __volatile("itr.d dtr[%0]=%1" :: "r"(2), "r"(*(u_int64_t*)&pte)); + __asm __volatile("mov psr.l=%0" :: "r" (psr)); + __asm __volatile("srlz.d"); +} + +void +map_gateway_page(void) +{ + struct ia64_pte pte; + u_int64_t psr; + + bzero(&pte, sizeof(pte)); + pte.pte_p = 1; + pte.pte_ma = PTE_MA_WB; + pte.pte_a = 1; + pte.pte_d = 1; + pte.pte_pl = PTE_PL_KERN; + pte.pte_ar = PTE_AR_X_RX; + pte.pte_ppn = IA64_RR_MASK((u_int64_t)ia64_gateway_page) >> 12; + + __asm __volatile("ptr.d %0,%1; ptr.i %0,%1" :: + "r"(VM_MAX_ADDRESS), "r"(PAGE_SHIFT << 2)); + + __asm __volatile("mov %0=psr" : "=r"(psr)); + __asm __volatile("rsm psr.ic|psr.i"); + __asm __volatile("srlz.i"); + __asm __volatile("mov cr.ifa=%0" :: "r"(VM_MAX_ADDRESS)); + __asm __volatile("mov cr.itir=%0" :: "r"(PAGE_SHIFT << 2)); + __asm __volatile("itr.d dtr[%0]=%1" :: "r"(3), "r"(*(u_int64_t*)&pte)); + __asm __volatile("srlz.d"); /* XXX not needed. */ + __asm __volatile("itr.i itr[%0]=%1" :: "r"(3), "r"(*(u_int64_t*)&pte)); + __asm __volatile("mov psr.l=%0" :: "r" (psr)); + __asm __volatile("srlz.i"); + + /* Expose the mapping to userland in ar.k5 */ + ia64_set_k5(VM_MAX_ADDRESS); } static void @@ -639,21 +720,12 @@ ia64_init(u_int64_t arg1, u_int64_t arg2) /* * Setup the global data for the bootstrap cpu. */ - pcpup = (struct pcpu *) pmap_steal_memory(PAGE_SIZE); + pcpup = (struct pcpu *)pmap_steal_memory(PAGE_SIZE); + ia64_set_k4((u_int64_t)pcpup); pcpu_init(pcpup, 0, PAGE_SIZE); - ia64_set_k4((u_int64_t) pcpup); PCPU_SET(curthread, &thread0); /* - * Set ia32 control registers. - */ - ia64_set_cflg((CR0_PE | CR0_PG) - | ((long)(CR4_XMM | CR4_FXSR) << 32)); - - /* We pretend to own FP state so that ia64_fpstate_check() works */ - PCPU_SET(fpcurthread, &thread0); - - /* * Initialize the rest of proc 0's PCB. * * Set the kernel sp, reserving space for an (empty) trapframe, @@ -663,8 +735,11 @@ ia64_init(u_int64_t arg1, u_int64_t arg2) * XXX what is all this +/- 16 stuff? */ thread0.td_frame = (struct trapframe *)thread0.td_pcb - 1; - thread0.td_pcb->pcb_sp = (u_int64_t)thread0.td_frame - 16; - thread0.td_pcb->pcb_ar_bsp = (u_int64_t)proc0kstack; + thread0.td_frame->tf_length = sizeof(struct trapframe); + thread0.td_frame->tf_flags = FRAME_SYSCALL; + thread0.td_pcb->pcb_special.sp = + (u_int64_t)thread0.td_frame - 16; + thread0.td_pcb->pcb_special.bspstore = (u_int64_t)proc0kstack; mutex_init(); @@ -732,40 +807,29 @@ DELAY(int n) } /* - * Send an interrupt to process. - * - * Stack is set up to allow sigcode stored - * at top to call routine, followed by kcall - * to sigreturn routine below. After sigreturn - * resets the signal mask, the stack, and the - * frame pointer, it returns to the user - * specified pc, psl. + * Send an interrupt (signal) to a process. */ void sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code) { struct proc *p; struct thread *td; - struct trapframe *frame; + struct trapframe *tf; struct sigacts *psp; struct sigframe sf, *sfp; - u_int64_t sbs = 0; - int oonstack, rndfsize; + mcontext_t *mc; + u_int64_t sbs, sp; + int oonstack; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); - frame = td->td_frame; - oonstack = sigonstack(frame->tf_r[FRAME_SP]); - rndfsize = ((sizeof(sf) + 15) / 16) * 16; - - /* - * Make sure that we restore the entire trapframe after a - * signal. - */ - frame->tf_flags &= ~FRAME_SYSCALL; + tf = td->td_frame; + sp = tf->tf_special.sp; + oonstack = sigonstack(sp); + sbs = 0; /* save user context */ bzero(&sf, sizeof(struct sigframe)); @@ -773,33 +837,6 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code) sf.sf_uc.uc_stack = p->p_sigstk; sf.sf_uc.uc_stack.ss_flags = (p->p_flag & P_ALTSTACK) ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE; - sf.sf_uc.uc_mcontext.mc_flags = IA64_MC_FLAG_ONSTACK; - sf.sf_uc.uc_mcontext.mc_onstack = (oonstack) ? 1 : 0; - - sf.sf_uc.uc_mcontext.mc_nat = 0; /* XXX */ - sf.sf_uc.uc_mcontext.mc_sp = frame->tf_r[FRAME_SP]; - sf.sf_uc.uc_mcontext.mc_ip = (frame->tf_cr_iip - | ((frame->tf_cr_ipsr >> 41) & 3)); - sf.sf_uc.uc_mcontext.mc_cfm = frame->tf_cr_ifs & ~(1<<31); - sf.sf_uc.uc_mcontext.mc_um = frame->tf_cr_ipsr & 0x1fff; - sf.sf_uc.uc_mcontext.mc_ar_rsc = frame->tf_ar_rsc; - sf.sf_uc.uc_mcontext.mc_ar_bsp = frame->tf_ar_bspstore; - sf.sf_uc.uc_mcontext.mc_ar_rnat = frame->tf_ar_rnat; - sf.sf_uc.uc_mcontext.mc_ar_ccv = frame->tf_ar_ccv; - sf.sf_uc.uc_mcontext.mc_ar_unat = frame->tf_ar_unat; - sf.sf_uc.uc_mcontext.mc_ar_fpsr = frame->tf_ar_fpsr; - sf.sf_uc.uc_mcontext.mc_ar_pfs = frame->tf_ar_pfs; - sf.sf_uc.uc_mcontext.mc_pr = frame->tf_pr; - - bcopy(&frame->tf_b[0], - &sf.sf_uc.uc_mcontext.mc_br[0], - 8 * sizeof(unsigned long)); - sf.sf_uc.uc_mcontext.mc_gr[0] = 0; - bcopy(&frame->tf_r[0], - &sf.sf_uc.uc_mcontext.mc_gr[1], - 31 * sizeof(unsigned long)); - - /* XXX mc_fr[] */ /* * Allocate and validate space for the signal handler @@ -810,47 +847,46 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code) */ if ((p->p_flag & P_ALTSTACK) != 0 && !oonstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { - sbs = (u_int64_t) p->p_sigstk.ss_sp; - sfp = (struct sigframe *)((caddr_t)p->p_sigstk.ss_sp + - p->p_sigstk.ss_size - rndfsize); - /* - * Align sp and bsp. - */ + sbs = (u_int64_t)p->p_sigstk.ss_sp; sbs = (sbs + 15) & ~15; - sfp = (struct sigframe *)((u_int64_t)sfp & ~15); + sfp = (struct sigframe *)(sbs + p->p_sigstk.ss_size); #if defined(COMPAT_43) || defined(COMPAT_SUNOS) p->p_sigstk.ss_flags |= SS_ONSTACK; #endif } else - sfp = (struct sigframe *)(frame->tf_r[FRAME_SP] - rndfsize); + sfp = (struct sigframe *)sp; + sfp = (struct sigframe *)((u_int64_t)(sfp - 1) & ~15); + + /* Fill in the siginfo structure for POSIX handlers. */ + if (SIGISMEMBER(psp->ps_siginfo, sig)) { + sf.sf_si.si_signo = sig; + sf.sf_si.si_code = code; + sf.sf_si.si_addr = (void*)tf->tf_special.ifa; + code = (u_int64_t)&sfp->sf_si; + } + mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(p); -#ifdef DEBUG - if ((sigdebug & SDB_KSTACK) && p->p_pid == sigpid) - printf("sendsig(%d): sig %d ssp %p usp %p\n", p->p_pid, - sig, &sf, sfp); -#endif - -#if 0 - /* save the floating-point state, if necessary, then copy it. */ - ia64_fpstate_save(td, 1); - sf.sf_uc.uc_mcontext.mc_ownedfp = td->td_md.md_flags & MDP_FPUSED; - bcopy(&td->td_pcb->pcb_fp, - (struct fpreg *)sf.sf_uc.uc_mcontext.mc_fpregs, - sizeof(struct fpreg)); - sf.sf_uc.uc_mcontext.mc_fp_control = td->td_pcb.pcb_fp_control; -#endif + mc = &sf.sf_uc.uc_mcontext; + mc->mc_special = tf->tf_special; + mc->mc_scratch = tf->tf_scratch; + if ((tf->tf_flags & FRAME_SYSCALL) == 0) { + mc->mc_flags |= IA64_MC_FLAGS_SCRATCH_VALID; + mc->mc_scratch_fp = tf->tf_scratch_fp; + /* + * XXX High FP. If the process has never used the high FP, + * mark the high FP as valid (zero defaults). If the process + * did use the high FP, then store them in the PCB if not + * already there (ie get them from the CPU that has them) + * and write them in the context. + */ + } + save_callee_saved(&mc->mc_preserved); + save_callee_saved_fp(&mc->mc_preserved_fp); - /* - * copy the frame out to userland. - */ - if (copyout((caddr_t)&sf, (caddr_t)sfp, sizeof(sf)) != 0) { -#ifdef DEBUG - if ((sigdebug & SDB_KSTACK) && p->p_pid == sigpid) - printf("sendsig(%d): copyout failed on sig %d\n", - p->p_pid, sig); -#endif + /* Copy the frame out to userland. */ + if (copyout(&sf, sfp, sizeof(sf)) != 0) { /* * Process has trashed its stack; give it an illegal * instruction to halt it in its tracks. @@ -859,46 +895,32 @@ sendsig(sig_t catcher, int sig, sigset_t *mask, u_long code) sigexit(td, SIGILL); return; } -#ifdef DEBUG - if (sigdebug & SDB_FOLLOW) - printf("sendsig(%d): sig %d sfp %p code %lx\n", p->p_pid, sig, - sfp, code); -#endif + + if ((tf->tf_flags & FRAME_SYSCALL) == 0) { + tf->tf_special.psr &= ~IA64_PSR_RI; + tf->tf_special.iip = ia64_get_k5() + + ((uint64_t)break_sigtramp - (uint64_t)ia64_gateway_page); + } else + tf->tf_special.rp = ia64_get_k5() + + ((uint64_t)epc_sigtramp - (uint64_t)ia64_gateway_page); /* - * Set up the registers to return to sigcode. + * Setup the trapframe to return to the signal trampoline. We pass + * information to the trampoline in the following registers: + * + * gp new backing store or NULL + * r8 signal number + * r9 signal code or siginfo pointer + * r10 signal handler (function descriptor) */ - frame->tf_cr_ipsr &= ~IA64_PSR_RI; - frame->tf_cr_iip = PS_STRINGS - (esigcode - sigcode); - frame->tf_r[FRAME_R1] = sig; + tf->tf_special.sp = (u_int64_t)sfp - 16; + tf->tf_special.gp = sbs; + tf->tf_scratch.gr8 = sig; + tf->tf_scratch.gr9 = code; + tf->tf_scratch.gr10 = (u_int64_t)catcher; + PROC_LOCK(p); mtx_lock(&psp->ps_mtx); - if (SIGISMEMBER(psp->ps_siginfo, sig)) { - frame->tf_r[FRAME_R15] = (u_int64_t)&(sfp->sf_si); - - /* Fill in POSIX parts */ - sf.sf_si.si_signo = sig; - sf.sf_si.si_code = code; - sf.sf_si.si_addr = (void*)frame->tf_cr_ifa; - } - else - frame->tf_r[FRAME_R15] = code; - - frame->tf_r[FRAME_SP] = (u_int64_t)sfp - 16; - frame->tf_r[FRAME_R14] = sig; - frame->tf_r[FRAME_R15] = (u_int64_t) &sfp->sf_si; - frame->tf_r[FRAME_R16] = (u_int64_t) &sfp->sf_uc; - frame->tf_r[FRAME_R17] = (u_int64_t)catcher; - frame->tf_r[FRAME_R18] = sbs; - -#ifdef DEBUG - if (sigdebug & SDB_FOLLOW) - printf("sendsig(%d): pc %lx, catcher %lx\n", p->p_pid, - frame->tf_cr_iip, frame->tf_regs[FRAME_R4]); - if ((sigdebug & SDB_KSTACK) && p->p_pid == sigpid) - printf("sendsig(%d): sig %d returns\n", - p->p_pid, sig); -#endif } /* @@ -919,92 +941,54 @@ sigreturn(struct thread *td, } */ *uap) { ucontext_t uc; - const ucontext_t *ucp; - struct pcb *pcb; - struct trapframe *frame = td->td_frame; - struct __mcontext *mcp; + struct trapframe *tf; + struct __mcontext *mc; struct proc *p; + struct pcb *pcb; - ucp = uap->sigcntxp; - pcb = td->td_pcb; + tf = td->td_frame; p = td->td_proc; - -#ifdef DEBUG - if (sigdebug & SDB_FOLLOW) - printf("sigreturn: pid %d, scp %p\n", p->p_pid, ucp); -#endif + pcb = td->td_pcb; /* * Fetch the entire context structure at once for speed. * We don't use a normal argument to simplify RSE handling. */ - if (copyin((caddr_t)frame->tf_r[FRAME_R4], - (caddr_t)&uc, sizeof(ucontext_t))) + if (copyin(uap->sigcntxp, (caddr_t)&uc, sizeof(uc))) return (EFAULT); - if (frame->tf_ndirty != 0) { - printf("sigreturn: dirty user stacked registers\n"); - } + /* + * XXX make sure ndirty in the current trapframe is less than + * 0x1f8 so that if we throw away the current register stack, + * we have reached the bottom of the kernel register stack. + * See also exec_setregs. + */ /* * Restore the user-supplied information */ - mcp = &uc.uc_mcontext; - bcopy(&mcp->mc_br[0], &frame->tf_b[0], 8*sizeof(u_int64_t)); - bcopy(&mcp->mc_gr[1], &frame->tf_r[0], 31*sizeof(u_int64_t)); - /* XXX mc_fr */ - - frame->tf_flags &= ~FRAME_SYSCALL; - frame->tf_cr_iip = mcp->mc_ip & ~15; - frame->tf_cr_ipsr &= ~IA64_PSR_RI; - switch (mcp->mc_ip & 15) { - case 1: - frame->tf_cr_ipsr |= IA64_PSR_RI_1; - break; - case 2: - frame->tf_cr_ipsr |= IA64_PSR_RI_2; - break; + mc = &uc.uc_mcontext; + tf->tf_special = mc->mc_special; + tf->tf_scratch = mc->mc_scratch; + if ((mc->mc_flags & IA64_MC_FLAGS_SCRATCH_VALID) != 0) { + tf->tf_scratch_fp = mc->mc_scratch_fp; + /* XXX high FP. */ } - frame->tf_cr_ipsr = ((frame->tf_cr_ipsr & ~0x1fff) - | (mcp->mc_um & 0x1fff)); - frame->tf_pr = mcp->mc_pr; - frame->tf_ar_rsc = (mcp->mc_ar_rsc & 3) | 12; /* user, loadrs=0 */ - frame->tf_ar_pfs = mcp->mc_ar_pfs; - frame->tf_cr_ifs = mcp->mc_cfm | (1UL<<63); - frame->tf_ar_bspstore = mcp->mc_ar_bsp; - frame->tf_ar_rnat = mcp->mc_ar_rnat; - frame->tf_ndirty = 0; /* assumes flushrs in sigcode */ - frame->tf_ar_unat = mcp->mc_ar_unat; - frame->tf_ar_ccv = mcp->mc_ar_ccv; - frame->tf_ar_fpsr = mcp->mc_ar_fpsr; - - frame->tf_r[FRAME_SP] = mcp->mc_sp; + restore_callee_saved(&mc->mc_preserved); + restore_callee_saved_fp(&mc->mc_preserved_fp); PROC_LOCK(p); #if defined(COMPAT_43) || defined(COMPAT_SUNOS) - if (uc.uc_mcontext.mc_onstack & 1) + if (sigonstack(tf->tf_special.sp)) p->p_sigstk.ss_flags |= SS_ONSTACK; else p->p_sigstk.ss_flags &= ~SS_ONSTACK; #endif - td->td_sigmask = uc.uc_sigmask; SIG_CANTMASK(td->td_sigmask); signotify(td); PROC_UNLOCK(p); - /* XXX ksc.sc_ownedfp ? */ - ia64_fpstate_drop(td); -#if 0 - bcopy((struct fpreg *)uc.uc_mcontext.mc_fpregs, - &td->td_pcb->pcb_fp, sizeof(struct fpreg)); - td->td_pcb->pcb_fp_control = uc.uc_mcontext.mc_fp_control; -#endif - -#ifdef DEBUG - if (sigdebug & SDB_FOLLOW) - printf("sigreturn(%d): returns\n", p->p_pid); -#endif return (EJUSTRETURN); } @@ -1052,56 +1036,84 @@ cpu_halt(void) } /* - * Clear registers on exec + * Clear registers on exec. */ void exec_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) { - struct trapframe *frame; + struct trapframe *tf; + uint64_t bspst, kstack, ndirty; + size_t rssz; - frame = td->td_frame; + tf = td->td_frame; + kstack = td->td_kstack; /* - * Make sure that we restore the entire trapframe after an - * execve. - */ - frame->tf_flags &= ~FRAME_SYSCALL; - - bzero(frame->tf_r, sizeof(frame->tf_r)); - bzero(frame->tf_f, sizeof(frame->tf_f)); - frame->tf_cr_iip = entry; - frame->tf_cr_ipsr = (IA64_PSR_IC - | IA64_PSR_I - | IA64_PSR_IT - | IA64_PSR_DT - | IA64_PSR_RT - | IA64_PSR_DFH - | IA64_PSR_BN - | IA64_PSR_CPL_USER); - /* - * Make sure that sp is aligned to a 16 byte boundary and - * reserve 16 bytes of scratch space for _start. + * RSE magic: We have ndirty registers of the process on the kernel + * stack which don't belong to the new image. Discard them. Note + * that for the "legacy" syscall support we need to keep 3 registers + * worth of dirty bytes. These 3 registers are the initial arguments + * to the newly executing program. + * However, we cannot discard all the ndirty registers by simply + * moving the kernel related registers to the bottom of the kernel + * stack and lowering the current bspstore, because we get into + * trouble with the NaT collections. We need to keep that in sync + * with the registers. Hence, we can only copy a multiple of 512 + * bytes. Consequently, we may end up with some registers of the + * previous image on the kernel stack. This we ignore by making + * sure we mask-off the lower 9 bits of the bspstore value just + * prior to saving it in ar.k6. */ - frame->tf_r[FRAME_SP] = (stack & ~15) - 16; + ndirty = tf->tf_special.ndirty & ~0x1ff; + if (ndirty > 0) { + __asm __volatile("mov ar.rsc=0;;"); + __asm __volatile("mov %0=ar.bspstore" : "=r"(bspst)); + rssz = bspst - kstack - ndirty; + bcopy((void*)(kstack + ndirty), (void*)kstack, rssz); + bspst -= ndirty; + __asm __volatile("mov ar.bspstore=%0;;" :: "r"(bspst)); + __asm __volatile("mov ar.rsc=3"); + tf->tf_special.ndirty -= ndirty; + } + ndirty = tf->tf_special.ndirty; - /* - * Write values for out0, out1 and out2 to the user's backing - * store and arrange for them to be restored into the user's - * initial register frame. Assumes that (bspstore & 0x1f8) < - * 0x1e0. - */ - frame->tf_ar_bspstore = td->td_md.md_bspstore + 24; - suword((caddr_t) frame->tf_ar_bspstore - 24, stack); - suword((caddr_t) frame->tf_ar_bspstore - 16, ps_strings); - suword((caddr_t) frame->tf_ar_bspstore - 8, 0); - frame->tf_ndirty = 0; - frame->tf_cr_ifs = (1L<<63) | 3; /* sof=3, v=1 */ - - frame->tf_ar_rsc = 0xf; /* user mode rsc */ - frame->tf_ar_fpsr = IA64_FPSR_DEFAULT; - - td->td_md.md_flags &= ~MDP_FPUSED; - ia64_fpstate_drop(td); + bzero(&tf->tf_special, sizeof(tf->tf_special)); + + if ((tf->tf_flags & FRAME_SYSCALL) == 0) { /* break syscalls. */ + bzero(&tf->tf_scratch, sizeof(tf->tf_scratch)); + bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp)); + tf->tf_special.iip = entry; + tf->tf_special.cfm = (1UL<<63) | (3UL<<7) | 3UL; + tf->tf_special.bspstore = td->td_md.md_bspstore; + tf->tf_special.ndirty = 24; + /* + * Copy the arguments onto the kernel register stack so that + * they get loaded by the loadrs instruction. + */ + *(uint64_t*)(kstack + ndirty - 24) = stack; + *(uint64_t*)(kstack + ndirty - 16) = ps_strings; + *(uint64_t*)(kstack + ndirty - 8) = 0; + } else { /* epc syscalls (default). */ + tf->tf_special.rp = entry; + tf->tf_special.pfs = (3UL<<62) | (3UL<<7) | 3UL; + tf->tf_special.bspstore = td->td_md.md_bspstore + 24; + /* + * Write values for out0, out1 and out2 to the user's backing + * store and arrange for them to be restored into the user's + * initial register frame. + * Assumes that (bspstore & 0x1f8) < 0x1e0. + */ + suword((caddr_t)tf->tf_special.bspstore - 24, stack); + suword((caddr_t)tf->tf_special.bspstore - 16, ps_strings); + suword((caddr_t)tf->tf_special.bspstore - 8, 0); + } + + tf->tf_special.sp = (stack & ~15) - 16; + tf->tf_special.rsc = 0xf; + tf->tf_special.fpsr = IA64_FPSR_DEFAULT; + tf->tf_special.psr = IA64_PSR_IC | IA64_PSR_I | IA64_PSR_IT | + IA64_PSR_DT | IA64_PSR_RT | IA64_PSR_DFH | IA64_PSR_BN | + IA64_PSR_CPL_USER; } int @@ -1124,16 +1136,17 @@ ptrace_set_pc(struct thread *td, unsigned long addr) return (EINVAL); } - td->td_frame->tf_cr_iip = addr & ~0x0FULL; - td->td_frame->tf_cr_ipsr = (td->td_frame->tf_cr_ipsr & ~IA64_PSR_RI) | - slot; + td->td_frame->tf_special.iip = addr & ~0x0FULL; + td->td_frame->tf_special.psr = + (td->td_frame->tf_special.psr & ~IA64_PSR_RI) | slot; return (0); } int ptrace_single_step(struct thread *td) { - td->td_frame->tf_cr_ipsr |= IA64_PSR_SS; + + td->td_frame->tf_special.psr |= IA64_PSR_SS; return (0); } @@ -1144,66 +1157,26 @@ ia64_pa_access(vm_offset_t pa) } int -fill_regs(td, regs) - struct thread *td; - struct reg *regs; +fill_regs(struct thread *td, struct reg *regs) { - bcopy(td->td_frame->tf_b, regs->r_br, sizeof(regs->r_br)); - bcopy(td->td_frame->tf_r, regs->r_gr+1, sizeof(td->td_frame->tf_r)); - /* TODO copy registers from the register stack. */ - - regs->r_cfm = td->td_frame->tf_cr_ifs; - regs->r_ip = td->td_frame->tf_cr_iip; - regs->r_ip |= (td->td_frame->tf_cr_ipsr & IA64_PSR_RI) >> 41; - regs->r_pr = td->td_frame->tf_pr; - regs->r_psr = td->td_frame->tf_cr_ipsr; - regs->r_ar_rsc = td->td_frame->tf_ar_rsc; - regs->r_ar_bsp = 0; /* XXX */ - regs->r_ar_bspstore = td->td_frame->tf_ar_bspstore; - regs->r_ar_rnat = td->td_frame->tf_ar_rnat; - regs->r_ar_ccv = td->td_frame->tf_ar_ccv; - regs->r_ar_unat = td->td_frame->tf_ar_unat; - regs->r_ar_fpsr = td->td_frame->tf_ar_fpsr; - regs->r_ar_pfs = td->td_frame->tf_ar_pfs; - regs->r_ar_lc = td->td_frame->tf_ar_lc; - regs->r_ar_ec = td->td_frame->tf_ar_ec; + struct trapframe *tf; + tf = td->td_frame; + regs->r_special = tf->tf_special; + regs->r_scratch = tf->tf_scratch; + /* XXX preserved */ return (0); } int -set_regs(td, regs) - struct thread *td; - struct reg *regs; +set_regs(struct thread *td, struct reg *regs) { - int error; - - error = ptrace_set_pc(td, regs->r_ip); - if (error) - return (error); - - td->td_frame->tf_cr_ipsr &= ~0x1FUL; /* clear user mask */ - td->td_frame->tf_cr_ipsr |= regs->r_psr & 0x1FUL; - - td->td_frame->tf_pr = regs->r_pr; - - /* XXX r_ar_bsp */ - - td->td_frame->tf_ar_rsc = regs->r_ar_rsc; - td->td_frame->tf_ar_pfs = regs->r_ar_pfs; - td->td_frame->tf_cr_ifs = regs->r_cfm; - td->td_frame->tf_ar_bspstore = regs->r_ar_bspstore; - td->td_frame->tf_ar_rnat = regs->r_ar_rnat; - td->td_frame->tf_ar_unat = regs->r_ar_unat; - td->td_frame->tf_ar_ccv = regs->r_ar_ccv; - td->td_frame->tf_ar_fpsr = regs->r_ar_fpsr; - td->td_frame->tf_ar_lc = regs->r_ar_lc; - td->td_frame->tf_ar_ec = regs->r_ar_ec; - - bcopy(regs->r_br, td->td_frame->tf_b, sizeof(td->td_frame->tf_b)); - bcopy(regs->r_gr+1, td->td_frame->tf_r, sizeof(td->td_frame->tf_r)); - /* TODO copy registers to the register stack. */ + struct trapframe *tf; + tf = td->td_frame; + tf->tf_special = regs->r_special; + tf->tf_scratch = regs->r_scratch; + /* XXX preserved */ return (0); } @@ -1222,67 +1195,105 @@ set_dbregs(struct thread *td, struct dbreg *dbregs) } int -fill_fpregs(td, fpregs) - struct thread *td; - struct fpreg *fpregs; +fill_fpregs(struct thread *td, struct fpreg *fpregs) { + struct trapframe *frame = td->td_frame; struct pcb *pcb = td->td_pcb; - /* - * XXX - The PCB pointer should not point to the actual PCB, - * because it will not contain the preserved registers of - * the program being debugged. Instead, it should point to - * a PCB constructed by unwinding all the way up to the - * IVT handler. - */ - bcopy(pcb->pcb_f + PCB_F2, fpregs->fpr_regs + 2, - sizeof(pcb->pcb_f[0]) * 4); + /* Save the high FP registers. */ + ia64_highfp_save(td); - bcopy(td->td_frame->tf_f, fpregs->fpr_regs + 6, - sizeof(td->td_frame->tf_f)); + fpregs->fpr_scratch = frame->tf_scratch_fp; + /* XXX preserved_fp */ + fpregs->fpr_high = pcb->pcb_high_fp; + return (0); +} - bcopy(pcb->pcb_f + PCB_F16, fpregs->fpr_regs + 16, - sizeof(pcb->pcb_f[0]) * 16); +int +set_fpregs(struct thread *td, struct fpreg *fpregs) +{ + struct trapframe *frame = td->td_frame; + struct pcb *pcb = td->td_pcb; - ia64_fpstate_save(td, 0); - bcopy(pcb->pcb_highfp, fpregs->fpr_regs + 32, - sizeof(td->td_pcb->pcb_highfp)); + /* Throw away the high FP registers (should be redundant). */ + ia64_highfp_drop(td); + frame->tf_scratch_fp = fpregs->fpr_scratch; + /* XXX preserved_fp */ + pcb->pcb_high_fp = fpregs->fpr_high; return (0); } +/* + * High FP register functions. + * XXX no synchronization yet. + */ + int -set_fpregs(td, fpregs) - struct thread *td; - struct fpreg *fpregs; +ia64_highfp_drop(struct thread *td) { - struct pcb *pcb = td->td_pcb; + struct pcb *pcb; + struct pcpu *cpu; + struct thread *thr; - /* - * XXX - The PCB pointer should not point to the actual PCB, - * because it will not contain the preserved registers of - * the program being debugged. Instead, it should point to - * a PCB constructed by unwinding all the way up to the - * IVT handler. - * XXX - An additional complication here is that we need to - * have the actual location of where the values should be - * stored as well. Some values may still reside in registers, - * while other may have been saved somewhere. - */ - bcopy(fpregs->fpr_regs + 2, pcb->pcb_f + PCB_F2, - sizeof(pcb->pcb_f[0]) * 4); + pcb = td->td_pcb; + cpu = pcb->pcb_fpcpu; + if (cpu == NULL) + return (0); + pcb->pcb_fpcpu = NULL; + thr = cpu->pc_fpcurthread; + cpu->pc_fpcurthread = NULL; + + /* Post-mortem sanity checking. */ + KASSERT(thr == td, ("Inconsistent high FP state")); + return (1); +} - bcopy(fpregs->fpr_regs + 6, td->td_frame->tf_f, - sizeof(td->td_frame->tf_f)); +int +ia64_highfp_load(struct thread *td) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + KASSERT(pcb->pcb_fpcpu == NULL, ("FP race on thread")); + KASSERT(PCPU_GET(fpcurthread) == NULL, ("FP race on pcpu")); + restore_high_fp(&pcb->pcb_high_fp); + PCPU_SET(fpcurthread, td); + pcb->pcb_fpcpu = pcpup; + return (1); +} - bcopy(fpregs->fpr_regs + 16, pcb->pcb_f + PCB_F16, - sizeof(pcb->pcb_f[0]) * 16); +int +ia64_highfp_save(struct thread *td) +{ + struct pcb *pcb; + struct pcpu *cpu; + struct thread *thr; - ia64_fpstate_drop(td); - bcopy(fpregs->fpr_regs + 32, pcb->pcb_highfp, - sizeof(td->td_pcb->pcb_highfp)); + /* Don't save if the high FP registers weren't modified. */ + if ((td->td_frame->tf_special.psr & IA64_PSR_MFH) == 0) + return (ia64_highfp_drop(td)); - return (0); + pcb = td->td_pcb; + cpu = pcb->pcb_fpcpu; + if (cpu == NULL) + return (0); +#ifdef SMP + if (cpu != pcpup) { + ipi_send(cpu->pc_lid, IPI_HIGH_FP); + while (pcb->pcb_fpcpu != cpu) + DELAY(100); + return (1); + } +#endif + save_high_fp(&pcb->pcb_high_fp); + pcb->pcb_fpcpu = NULL; + thr = cpu->pc_fpcurthread; + cpu->pc_fpcurthread = NULL; + + /* Post-mortem sanity cxhecking. */ + KASSERT(thr == td, ("Inconsistent high FP state")); + return (1); } #ifndef DDB @@ -1313,76 +1324,6 @@ SYSCTL_INT(_machdep, CPU_DISRTCSET, disable_rtc_set, SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock, CTLFLAG_RW, &wall_cmos_clock, 0, ""); -void -ia64_fpstate_check(struct thread *td) -{ - if ((td->td_frame->tf_cr_ipsr & IA64_PSR_DFH) == 0) - if (td != PCPU_GET(fpcurthread)) - panic("ia64_fpstate_check: bogus"); -} - -/* - * Save the high floating point state in the pcb. Use this to get - * read-only access to the floating point state. If write is true, the - * current fp process is cleared so that fp state can safely be - * modified. The process will automatically reload the changed state - * by generating a disabled fp trap. - */ -void -ia64_fpstate_save(struct thread *td, int write) -{ - if (td == PCPU_GET(fpcurthread)) { - /* - * Save the state in the pcb. - */ - savehighfp(td->td_pcb->pcb_highfp); - - if (write) { - td->td_frame->tf_cr_ipsr |= IA64_PSR_DFH; - PCPU_SET(fpcurthread, NULL); - } - } -} - -/* - * Relinquish ownership of the FP state. This is called instead of - * ia64_save_fpstate() if the entire FP state is being changed - * (e.g. on sigreturn). - */ -void -ia64_fpstate_drop(struct thread *td) -{ - if (td == PCPU_GET(fpcurthread)) { - td->td_frame->tf_cr_ipsr |= IA64_PSR_DFH; - PCPU_SET(fpcurthread, NULL); - } -} - -/* - * Switch the current owner of the fp state to p, reloading the state - * from the pcb. - */ -void -ia64_fpstate_switch(struct thread *td) -{ - if (PCPU_GET(fpcurthread)) { - /* - * Dump the old fp state if its valid. - */ - savehighfp(PCPU_GET(fpcurthread)->td_pcb->pcb_highfp); - PCPU_GET(fpcurthread)->td_frame->tf_cr_ipsr |= IA64_PSR_DFH; - } - - /* - * Remember the new FP owner and reload its state. - */ - PCPU_SET(fpcurthread, td); - restorehighfp(td->td_pcb->pcb_highfp); - td->td_frame->tf_cr_ipsr &= ~IA64_PSR_DFH; - - td->td_md.md_flags |= MDP_FPUSED; -} - /* * Utility functions for manipulating instruction bundles. */ @@ -1407,47 +1348,8 @@ ia64_pack_bundle(u_int64_t *lowp, u_int64_t *highp, *highp = high; } -static int -rse_slot(u_int64_t *bsp) -{ - return ((u_int64_t) bsp >> 3) & 0x3f; -} - -/* - * Return the address of register regno (regno >= 32) given that bsp - * points at the base of the register stack frame. - */ -u_int64_t * -ia64_rse_register_address(u_int64_t *bsp, int regno) -{ - int off = regno - 32; - u_int64_t rnats = (rse_slot(bsp) + off) / 63; - return bsp + off + rnats; -} - -/* - * Calculate the base address of the previous frame given that the - * current frame's locals area is 'size'. - */ -u_int64_t * -ia64_rse_previous_frame(u_int64_t *bsp, int size) -{ - int slot = rse_slot(bsp); - int rnats = 0; - int count = size; - - while (count > slot) { - count -= 63; - rnats++; - slot = 63; - } - return bsp - size - rnats; -} - - intptr_t casuptr(intptr_t *p, intptr_t old, intptr_t new) { return (-1); } - diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c index 43c06a7..f06e0cc 100644 --- a/sys/ia64/ia64/mp_machdep.c +++ b/sys/ia64/ia64/mp_machdep.c @@ -56,6 +56,8 @@ #include <machine/fpu.h> #include <i386/include/specialreg.h> +MALLOC_DECLARE(M_PMAP); + void ia64_ap_startup(void); extern vm_offset_t vhpt_base, vhpt_size; @@ -69,36 +71,37 @@ extern u_int64_t ia64_lapic_address; int mp_ipi_test = 0; /* Variables used by os_boot_rendez */ -volatile vm_offset_t ap_stack; -volatile struct pcpu *ap_pcpu; +void *ap_stack; +struct pcpu *ap_pcpu; volatile int ap_delay; volatile int ap_awake; volatile int ap_spin; -static void ipi_send(u_int64_t, int); static void cpu_mp_unleash(void *); void ia64_ap_startup(void) { + ap_awake = 1; + ap_delay = 0; + __asm __volatile("mov cr.pta=%0;; srlz.i;;" :: "r" (vhpt_base + (1<<8) + (vhpt_size<<2) + 1)); - ia64_set_fpsr(IA64_FPSR_DEFAULT); + pcpup = ap_pcpu; + ia64_set_k4((intptr_t)pcpup); - /* - * Set ia32 control registers. - */ - ia64_set_cflg(CR0_PE | CR0_PG | ((long)(CR4_XMM|CR4_FXSR) << 32)); + map_pal_code(); + map_port_space(); + map_gateway_page(); - ap_awake = 1; - ap_delay = 0; + ia64_set_fpsr(IA64_FPSR_DEFAULT); /* Wait until it's time for us to be unleashed */ while (ap_spin) /* spin */; - __asm __volatile("ssm psr.ic|psr.i;; srlz.i;;"); + __asm __volatile("ssm psr.i;; srlz.d;;"); /* * Get and save the CPU specific MCA records. Should we get the @@ -122,8 +125,8 @@ ia64_ap_startup(void) ia64_set_itm(ia64_get_itc() + itm_reload); ia64_set_itv(CLOCK_VECTOR); ia64_set_tpr(0); - cpu_throw(); - panic("ia64_ap_startup: cpu_throw() returned"); + cpu_throw(NULL, choosethread()); + /* NOTREACHED */ } int @@ -213,22 +216,8 @@ cpu_mp_start() pc->pc_current_pmap = kernel_pmap; pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask; if (pc->pc_cpuid > 0) { - void *ks; - - /* - * Use contigmalloc for stack so that we can - * use a region 7 address for it which makes - * it impossible to accidentally lose when - * recording a trapframe. - */ - ks = contigmalloc(KSTACK_PAGES * PAGE_SIZE, M_TEMP, - M_WAITOK, - 0ul, - 256*1024*1024 - 1, - PAGE_SIZE, - 256*1024*1024); - - ap_stack = IA64_PHYS_TO_RR7(ia64_tpa((u_int64_t)ks)); + ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP, + M_WAITOK); ap_pcpu = pc; ap_delay = 2000; ap_awake = 0; @@ -344,7 +333,7 @@ ipi_self(int ipi) * cr.lid (CR64) contents of the target processor. Only the id and eid * fields are used here. */ -static void +void ipi_send(u_int64_t lid, int ipi) { volatile u_int64_t *pipi; diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index dcb49b9..998d49f 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -121,9 +121,13 @@ #include <sys/user.h> +#include <machine/cpu.h> #include <machine/pal.h> #include <machine/md_var.h> +/* XXX move to a header. */ +extern u_int64_t ia64_gateway_page[]; + MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures"); #ifndef KSTACK_MAX_PAGES @@ -201,7 +205,7 @@ vm_offset_t vhpt_base, vhpt_size; * ia64_lptes. This gives us up to 2Gb of kernel virtual space. */ static int nkpt; -static struct ia64_lpte **kptdir; +struct ia64_lpte **ia64_kptdir; #define KPTE_DIR_INDEX(va) \ ((va >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1)) #define KPTE_PTE_INDEX(va) \ @@ -369,12 +373,13 @@ pmap_bootstrap() /* * Allocate some memory for initial kernel 'page tables'. */ - kptdir = (struct ia64_lpte **) pmap_steal_memory(PAGE_SIZE); + ia64_kptdir = (void *)pmap_steal_memory(PAGE_SIZE); for (i = 0; i < NKPT; i++) { - kptdir[i] = (struct ia64_lpte *) pmap_steal_memory(PAGE_SIZE); + ia64_kptdir[i] = (void*)pmap_steal_memory(PAGE_SIZE); } nkpt = NKPT; - kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS; + kernel_vm_end = NKPT * PAGE_SIZE * NKPTEPG + VM_MIN_KERNEL_ADDRESS - + VM_GATEWAY_SIZE; avail_start = phys_avail[0]; for (i = 0; phys_avail[i+2]; i+= 2) ; @@ -497,6 +502,8 @@ pmap_bootstrap() * Clear out any random TLB entries left over from booting. */ pmap_invalidate_all(kernel_pmap); + + map_gateway_page(); } void * @@ -754,27 +761,14 @@ pmap_track_modified(vm_offset_t va) void pmap_new_thread(struct thread *td, int pages) { - vm_offset_t *ks; /* Bounds check */ if (pages <= 1) pages = KSTACK_PAGES; else if (pages > KSTACK_MAX_PAGES) pages = KSTACK_MAX_PAGES; - - /* - * Use contigmalloc for user area so that we can use a region - * 7 address for it which makes it impossible to accidentally - * lose when recording a trapframe. - */ - ks = contigmalloc(pages * PAGE_SIZE, M_PMAP, M_WAITOK, 0ul, - 256*1024*1024 - 1, PAGE_SIZE, 256*1024*1024); - if (ks == NULL) - panic("pmap_new_thread: could not contigmalloc %d pages\n", - pages); - - td->td_md.md_kstackvirt = ks; - td->td_kstack = IA64_PHYS_TO_RR7(ia64_tpa((u_int64_t)ks)); + td->td_kstack = (vm_offset_t)malloc(pages * PAGE_SIZE, M_PMAP, + M_WAITOK); td->td_kstack_pages = pages; } @@ -785,12 +779,10 @@ pmap_new_thread(struct thread *td, int pages) void pmap_dispose_thread(struct thread *td) { - int pages; - pages = td->td_kstack_pages; - contigfree(td->td_md.md_kstackvirt, pages * PAGE_SIZE, M_PMAP); - td->td_md.md_kstackvirt = NULL; + free((void*)td->td_kstack, M_PMAP); td->td_kstack = 0; + td->td_kstack_pages = 0; } /* @@ -800,16 +792,9 @@ void pmap_new_altkstack(struct thread *td, int pages) { - /* - * Shuffle the original stack. Save the virtual kstack address - * instead of the physical address because 1) we can derive the - * physical address from the virtual address and 2) we need the - * virtual address in pmap_dispose_thread. - */ + td->td_altkstack = td->td_kstack; td->td_altkstack_obj = td->td_kstack_obj; - td->td_altkstack = (vm_offset_t)td->td_md.md_kstackvirt; td->td_altkstack_pages = td->td_kstack_pages; - pmap_new_thread(td, pages); } @@ -818,13 +803,7 @@ pmap_dispose_altkstack(struct thread *td) { pmap_dispose_thread(td); - - /* - * Restore the original kstack. Note that td_altkstack holds the - * virtual kstack address of the previous kstack. - */ - td->td_md.md_kstackvirt = (void*)td->td_altkstack; - td->td_kstack = IA64_PHYS_TO_RR7(ia64_tpa(td->td_altkstack)); + td->td_kstack = td->td_altkstack; td->td_kstack_obj = td->td_altkstack_obj; td->td_kstack_pages = td->td_altkstack_pages; td->td_altkstack = 0; @@ -938,7 +917,7 @@ pmap_growkernel(vm_offset_t addr) ptepage = (struct ia64_lpte *) IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(nkpg)); bzero(ptepage, PAGE_SIZE); - kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage; + ia64_kptdir[KPTE_DIR_INDEX(kernel_vm_end)] = ptepage; nkpt++; kernel_vm_end += PAGE_SIZE * NKPTEPG; @@ -1210,7 +1189,7 @@ pmap_find_kpte(vm_offset_t va) ("kernel mapping 0x%lx not in region 5", va)); KASSERT(IA64_RR_MASK(va) < (nkpt * PAGE_SIZE * NKPTEPG), ("kernel mapping 0x%lx out of range", va)); - return &kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]; + return (&ia64_kptdir[KPTE_DIR_INDEX(va)][KPTE_PTE_INDEX(va)]); } /* @@ -1355,6 +1334,7 @@ vm_paddr_t pmap_kextract(vm_offset_t va) { struct ia64_lpte *pte; + vm_offset_t gwpage; KASSERT(va >= IA64_RR_BASE(5), ("Must be kernel VA")); @@ -1362,6 +1342,11 @@ pmap_kextract(vm_offset_t va) if (va >= IA64_RR_BASE(6)) return (IA64_RR_MASK(va)); + /* EPC gateway page? */ + gwpage = (vm_offset_t)ia64_get_k5(); + if (va >= gwpage && va < gwpage + VM_GATEWAY_SIZE) + return (IA64_RR_MASK((vm_offset_t)ia64_gateway_page)); + /* Bail out if the virtual address is beyond our limits. */ if (IA64_RR_MASK(va) >= nkpt * PAGE_SIZE * NKPTEPG) return (0); diff --git a/sys/ia64/ia64/support.S b/sys/ia64/ia64/support.S index 18537f0..7403a01 100644 --- a/sys/ia64/ia64/support.S +++ b/sys/ia64/ia64/support.S @@ -69,7 +69,6 @@ * ar.bsp tranlated to new mode */ ENTRY(ia64_change_mode, 0) - rsm psr.i | psr.ic mov r19=ar.rsc // save rsc while we change mode tbit.nz p6,p7=r14,17 // physical or virtual ? @@ -106,7 +105,6 @@ ENTRY(ia64_change_mode, 0) 2: mov ar.rsc=r19 // restore ar.rsc br.ret.sptk.few rp // now in new mode - END(ia64_change_mode) /* @@ -121,7 +119,6 @@ END(ia64_change_mode) * psr.i cleared */ ENTRY(ia64_physical_mode, 0) - mov r14=psr mov ret0=psr movl r15=(IA64_PSR_I|IA64_PSR_IT|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFL|IA64_PSR_DFH) @@ -133,7 +130,6 @@ ENTRY(ia64_physical_mode, 0) or ret0=ret0,r16 // make sure BN=1 br.cond.sptk.many ia64_change_mode - END(ia64_physical_mode) /* @@ -148,7 +144,6 @@ END(ia64_physical_mode) * */ ENTRY(ia64_call_efi_physical, 6) - .prologue .regstk 6,4,5,0 .save ar.pfs,loc0 @@ -183,7 +178,6 @@ ENTRY(ia64_call_efi_physical, 6) mov ar.pfs=loc0 ;; br.ret.sptk.many rp - END(ia64_call_efi_physical) /**************************************************************************/ @@ -194,8 +188,7 @@ END(ia64_call_efi_physical) ENTRY(suword64, 2) XENTRY(suword) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -218,12 +211,10 @@ XENTRY(suword) mov ret0=r0 br.ret.sptk.few rp - END(suword64) ENTRY(suword32, 2) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -246,12 +237,10 @@ ENTRY(suword32, 2) mov ret0=r0 br.ret.sptk.few rp - END(suword32) ENTRY(subyte, 2) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -274,13 +263,11 @@ ENTRY(subyte, 2) mov ret0=r0 br.ret.sptk.few rp - END(subyte) ENTRY(fuword64, 1) XENTRY(fuword) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -302,12 +289,10 @@ XENTRY(fuword) st8 [r15]=r0 // clean up br.ret.sptk.few rp - END(fuword64) ENTRY(fuword32, 1) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -329,12 +314,10 @@ ENTRY(fuword32, 1) st8 [r15]=r0 // clean up br.ret.sptk.few rp - END(fuword32) ENTRY(fubyte, 1) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -356,7 +339,6 @@ ENTRY(fubyte, 1) st8 [r15]=r0 // clean up br.ret.sptk.few rp - END(fubyte) ENTRY(fusufault, 0) @@ -410,7 +392,6 @@ ENTRY(copystr, 4) 4: mov ret0=0 // return 0. br.ret.sptk.few rp - END(copystr) ENTRY(copyinstr, 4) @@ -422,7 +403,7 @@ ENTRY(copyinstr, 4) mov loc1=rp .body - movl loc2=VM_MAXUSER_ADDRESS // make sure that src addr + movl loc2=VM_MAX_ADDRESS // make sure that src addr ;; cmp.geu p6,p0=in0,loc2 // is in user space. ;; @@ -450,54 +431,12 @@ ENTRY(copyinstr, 4) mov ar.pfs=loc0 // restore ar.pfs mov rp=loc1 // restore ra. br.ret.sptk.few rp // ret0 left over from copystr - END(copyinstr) -ENTRY(copyoutstr, 4) - .prologue - .regstk 4, 3, 4, 0 - .save ar.pfs,loc0 - alloc loc0=ar.pfs,4,3,4,0 - .save rp,loc1 - mov loc1=rp - .body - - movl loc2=VM_MAXUSER_ADDRESS // make sure that dest addr - ;; - cmp.geu p6,p0=in1,loc2 // is in user space. - ;; -(p6) br.cond.spnt.few copyerr // if it's not, error out. - movl r14=copyerr // set up fault handler. - add r15=PC_CURTHREAD,r13 // find curthread - ;; - ld8 r15=[r15] - ;; - add r15=TD_PCB,r15 // find pcb - ;; - ld8 r15=[r15] - ;; - add loc2=PCB_ONFAULT,r15 - ;; - st8 [loc2]=r14 - ;; - mov out0=in0 - mov out1=in1 - mov out2=in2 - mov out3=in3 - ;; - br.call.sptk.few rp=copystr // do the copy. - st8 [loc2]=r0 // kill the fault handler. - mov ar.pfs=loc0 // restore ar.pfs - mov rp=loc1 // restore ra. - br.ret.sptk.few rp // ret0 left over from copystr - -END(copyoutstr) - /* * Not the fastest bcopy in the world. */ ENTRY(bcopy, 3) - mov ret0=r0 // return zero for copy{in,out} ;; cmp.le p6,p0=in2,r0 // bail if len <= 0 @@ -559,29 +498,25 @@ ENTRY(bcopy, 3) (p6) br.cond.spnt.few 6b br.ret.sptk.few rp - END(bcopy) ENTRY(memcpy,3) - mov r14=in0 ;; mov in0=in1 ;; mov in1=r14 br.cond.sptk.few bcopy - END(memcpy) ENTRY(copyin, 3) - .prologue - .regstk 4, 3, 4, 0 + .regstk 3, 3, 3, 0 .save ar.pfs,loc0 - alloc loc0=ar.pfs,4,3,4,0 + alloc loc0=ar.pfs,3,3,3,0 .save rp,loc1 mov loc1=rp .body - movl loc2=VM_MAXUSER_ADDRESS // make sure that src addr + movl loc2=VM_MAX_ADDRESS // make sure that src addr ;; cmp.geu p6,p0=in0,loc2 // is in user space. ;; @@ -608,20 +543,18 @@ ENTRY(copyin, 3) mov ar.pfs=loc0 // restore ar.pfs mov rp=loc1 // restore ra. br.ret.sptk.few rp // ret0 left over from bcopy - END(copyin) ENTRY(copyout, 3) - .prologue - .regstk 4, 3, 4, 0 + .regstk 3, 3, 3, 0 .save ar.pfs,loc0 - alloc loc0=ar.pfs,4,3,4,0 + alloc loc0=ar.pfs,3,3,3,0 .save rp,loc1 mov loc1=rp .body - movl loc2=VM_MAXUSER_ADDRESS // make sure that dest addr + movl loc2=VM_MAX_ADDRESS // make sure that dest addr ;; cmp.geu p6,p0=in1,loc2 // is in user space. ;; @@ -648,11 +581,9 @@ ENTRY(copyout, 3) mov ar.pfs=loc0 // restore ar.pfs mov rp=loc1 // restore ra. br.ret.sptk.few rp // ret0 left over from bcopy - END(copyout) ENTRY(copyerr, 0) - add r14=PC_CURTHREAD,r13 ;; // find curthread ld8 r14=[r14] ;; add r14=TD_PCB,r14 ;; // curthread->td_addr @@ -662,5 +593,4 @@ ENTRY(copyerr, 0) mov ret0=EFAULT // return EFAULT br.ret.sptk.few rp - END(copyerr) diff --git a/sys/ia64/ia64/support.s b/sys/ia64/ia64/support.s index 18537f0..7403a01 100644 --- a/sys/ia64/ia64/support.s +++ b/sys/ia64/ia64/support.s @@ -69,7 +69,6 @@ * ar.bsp tranlated to new mode */ ENTRY(ia64_change_mode, 0) - rsm psr.i | psr.ic mov r19=ar.rsc // save rsc while we change mode tbit.nz p6,p7=r14,17 // physical or virtual ? @@ -106,7 +105,6 @@ ENTRY(ia64_change_mode, 0) 2: mov ar.rsc=r19 // restore ar.rsc br.ret.sptk.few rp // now in new mode - END(ia64_change_mode) /* @@ -121,7 +119,6 @@ END(ia64_change_mode) * psr.i cleared */ ENTRY(ia64_physical_mode, 0) - mov r14=psr mov ret0=psr movl r15=(IA64_PSR_I|IA64_PSR_IT|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFL|IA64_PSR_DFH) @@ -133,7 +130,6 @@ ENTRY(ia64_physical_mode, 0) or ret0=ret0,r16 // make sure BN=1 br.cond.sptk.many ia64_change_mode - END(ia64_physical_mode) /* @@ -148,7 +144,6 @@ END(ia64_physical_mode) * */ ENTRY(ia64_call_efi_physical, 6) - .prologue .regstk 6,4,5,0 .save ar.pfs,loc0 @@ -183,7 +178,6 @@ ENTRY(ia64_call_efi_physical, 6) mov ar.pfs=loc0 ;; br.ret.sptk.many rp - END(ia64_call_efi_physical) /**************************************************************************/ @@ -194,8 +188,7 @@ END(ia64_call_efi_physical) ENTRY(suword64, 2) XENTRY(suword) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -218,12 +211,10 @@ XENTRY(suword) mov ret0=r0 br.ret.sptk.few rp - END(suword64) ENTRY(suword32, 2) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -246,12 +237,10 @@ ENTRY(suword32, 2) mov ret0=r0 br.ret.sptk.few rp - END(suword32) ENTRY(subyte, 2) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -274,13 +263,11 @@ ENTRY(subyte, 2) mov ret0=r0 br.ret.sptk.few rp - END(subyte) ENTRY(fuword64, 1) XENTRY(fuword) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -302,12 +289,10 @@ XENTRY(fuword) st8 [r15]=r0 // clean up br.ret.sptk.few rp - END(fuword64) ENTRY(fuword32, 1) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -329,12 +314,10 @@ ENTRY(fuword32, 1) st8 [r15]=r0 // clean up br.ret.sptk.few rp - END(fuword32) ENTRY(fubyte, 1) - - movl r14=VM_MAXUSER_ADDRESS;; // make sure address is ok + movl r14=VM_MAX_ADDRESS;; // make sure address is ok cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few fusufault @@ -356,7 +339,6 @@ ENTRY(fubyte, 1) st8 [r15]=r0 // clean up br.ret.sptk.few rp - END(fubyte) ENTRY(fusufault, 0) @@ -410,7 +392,6 @@ ENTRY(copystr, 4) 4: mov ret0=0 // return 0. br.ret.sptk.few rp - END(copystr) ENTRY(copyinstr, 4) @@ -422,7 +403,7 @@ ENTRY(copyinstr, 4) mov loc1=rp .body - movl loc2=VM_MAXUSER_ADDRESS // make sure that src addr + movl loc2=VM_MAX_ADDRESS // make sure that src addr ;; cmp.geu p6,p0=in0,loc2 // is in user space. ;; @@ -450,54 +431,12 @@ ENTRY(copyinstr, 4) mov ar.pfs=loc0 // restore ar.pfs mov rp=loc1 // restore ra. br.ret.sptk.few rp // ret0 left over from copystr - END(copyinstr) -ENTRY(copyoutstr, 4) - .prologue - .regstk 4, 3, 4, 0 - .save ar.pfs,loc0 - alloc loc0=ar.pfs,4,3,4,0 - .save rp,loc1 - mov loc1=rp - .body - - movl loc2=VM_MAXUSER_ADDRESS // make sure that dest addr - ;; - cmp.geu p6,p0=in1,loc2 // is in user space. - ;; -(p6) br.cond.spnt.few copyerr // if it's not, error out. - movl r14=copyerr // set up fault handler. - add r15=PC_CURTHREAD,r13 // find curthread - ;; - ld8 r15=[r15] - ;; - add r15=TD_PCB,r15 // find pcb - ;; - ld8 r15=[r15] - ;; - add loc2=PCB_ONFAULT,r15 - ;; - st8 [loc2]=r14 - ;; - mov out0=in0 - mov out1=in1 - mov out2=in2 - mov out3=in3 - ;; - br.call.sptk.few rp=copystr // do the copy. - st8 [loc2]=r0 // kill the fault handler. - mov ar.pfs=loc0 // restore ar.pfs - mov rp=loc1 // restore ra. - br.ret.sptk.few rp // ret0 left over from copystr - -END(copyoutstr) - /* * Not the fastest bcopy in the world. */ ENTRY(bcopy, 3) - mov ret0=r0 // return zero for copy{in,out} ;; cmp.le p6,p0=in2,r0 // bail if len <= 0 @@ -559,29 +498,25 @@ ENTRY(bcopy, 3) (p6) br.cond.spnt.few 6b br.ret.sptk.few rp - END(bcopy) ENTRY(memcpy,3) - mov r14=in0 ;; mov in0=in1 ;; mov in1=r14 br.cond.sptk.few bcopy - END(memcpy) ENTRY(copyin, 3) - .prologue - .regstk 4, 3, 4, 0 + .regstk 3, 3, 3, 0 .save ar.pfs,loc0 - alloc loc0=ar.pfs,4,3,4,0 + alloc loc0=ar.pfs,3,3,3,0 .save rp,loc1 mov loc1=rp .body - movl loc2=VM_MAXUSER_ADDRESS // make sure that src addr + movl loc2=VM_MAX_ADDRESS // make sure that src addr ;; cmp.geu p6,p0=in0,loc2 // is in user space. ;; @@ -608,20 +543,18 @@ ENTRY(copyin, 3) mov ar.pfs=loc0 // restore ar.pfs mov rp=loc1 // restore ra. br.ret.sptk.few rp // ret0 left over from bcopy - END(copyin) ENTRY(copyout, 3) - .prologue - .regstk 4, 3, 4, 0 + .regstk 3, 3, 3, 0 .save ar.pfs,loc0 - alloc loc0=ar.pfs,4,3,4,0 + alloc loc0=ar.pfs,3,3,3,0 .save rp,loc1 mov loc1=rp .body - movl loc2=VM_MAXUSER_ADDRESS // make sure that dest addr + movl loc2=VM_MAX_ADDRESS // make sure that dest addr ;; cmp.geu p6,p0=in1,loc2 // is in user space. ;; @@ -648,11 +581,9 @@ ENTRY(copyout, 3) mov ar.pfs=loc0 // restore ar.pfs mov rp=loc1 // restore ra. br.ret.sptk.few rp // ret0 left over from bcopy - END(copyout) ENTRY(copyerr, 0) - add r14=PC_CURTHREAD,r13 ;; // find curthread ld8 r14=[r14] ;; add r14=TD_PCB,r14 ;; // curthread->td_addr @@ -662,5 +593,4 @@ ENTRY(copyerr, 0) mov ret0=EFAULT // return EFAULT br.ret.sptk.few rp - END(copyerr) diff --git a/sys/ia64/ia64/trap.c b/sys/ia64/ia64/trap.c index bccba0e..cad006b 100644 --- a/sys/ia64/ia64/trap.c +++ b/sys/ia64/ia64/trap.c @@ -46,6 +46,7 @@ #include <sys/sysent.h> #include <sys/syscall.h> #include <sys/pioctl.h> +#include <sys/sysctl.h> #include <vm/vm.h> #include <vm/vm_kern.h> #include <vm/vm_page.h> @@ -61,6 +62,9 @@ #include <machine/pal.h> #include <machine/fpu.h> #include <machine/efi.h> +#ifdef SMP +#include <machine/smp.h> +#endif #ifdef KTRACE #include <sys/uio.h> @@ -71,7 +75,13 @@ #include <ddb/ddb.h> #endif +static int print_usertrap = 0; +SYSCTL_INT(_machdep, CPU_UNALIGNED_PRINT, print_usertrap, + CTLFLAG_RW, &print_usertrap, 0, ""); + extern int unaligned_fixup(struct trapframe *framep, struct thread *td); + +static void break_syscall(struct trapframe *tf); static void ia32_syscall(struct trapframe *framep); /* @@ -93,12 +103,12 @@ typedef struct { * these there. The rest of the registers are "live" */ typedef struct { - u_int64_t bitmask_low64; /* f63 - f2 */ - u_int64_t bitmask_high64; /* f127 - f64 */ - struct ia64_fpreg *fp_low_preserved; /* f2 - f5 */ - struct ia64_fpreg *fp_low_volatile; /* f6 - f15 */ - struct ia64_fpreg *fp_high_preserved; /* f16 - f31 */ - struct ia64_fpreg *fp_high_volatile; /* f32 - f127 */ + u_int64_t bitmask_low64; /* f63 - f2 */ + u_int64_t bitmask_high64; /* f127 - f64 */ + struct _ia64_fpreg *fp_low_preserved; /* f2 - f5 */ + struct _ia64_fpreg *fp_low_volatile; /* f6 - f15 */ + struct _ia64_fpreg *fp_high_preserved; /* f16 - f31 */ + struct _ia64_fpreg *fp_high_volatile; /* f32 - f127 */ } FP_STATE; #ifdef WITNESS @@ -271,7 +281,7 @@ static void printisr(u_int64_t isr) } static void -printtrap(int vector, int imm, struct trapframe *framep, int isfatal, int user) +printtrap(int vector, struct trapframe *framep, int isfatal, int user) { printf("\n"); printf("%s %s trap (cpu %d):\n", isfatal? "fatal" : "handled", @@ -279,16 +289,15 @@ printtrap(int vector, int imm, struct trapframe *framep, int isfatal, int user) printf("\n"); printf(" trap vector = 0x%x (%s)\n", vector, ia64_vector_names[vector]); - printf(" cr.iip = 0x%lx\n", framep->tf_cr_iip); - printf(" cr.ipsr = 0x%lx (", framep->tf_cr_ipsr); - printpsr(framep->tf_cr_ipsr); + printf(" cr.iip = 0x%lx\n", framep->tf_special.iip); + printf(" cr.ipsr = 0x%lx (", framep->tf_special.psr); + printpsr(framep->tf_special.psr); printf(")\n"); - printf(" cr.isr = 0x%lx (", framep->tf_cr_isr); - printisr(framep->tf_cr_isr); + printf(" cr.isr = 0x%lx (", framep->tf_special.isr); + printisr(framep->tf_special.isr); printf(")\n"); - printf(" cr.ifa = 0x%lx\n", framep->tf_cr_ifa); - printf(" cr.iim = 0x%x\n", imm); - if (framep->tf_cr_ipsr & IA64_PSR_IS) { + printf(" cr.ifa = 0x%lx\n", framep->tf_special.ifa); + if (framep->tf_special.psr & IA64_PSR_IS) { printf(" ar.cflg = 0x%lx\n", ia64_get_cflg()); printf(" ar.csd = 0x%lx\n", ia64_get_csd()); printf(" ar.ssd = 0x%lx\n", ia64_get_ssd()); @@ -301,34 +310,57 @@ printtrap(int vector, int imm, struct trapframe *framep, int isfatal, int user) } /* + * + */ +int +do_ast(struct trapframe *tf) +{ + + disable_intr(); + while (curthread->td_flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)) { + enable_intr(); + ast(tf); + disable_intr(); + } + /* + * Keep interrupts disabled. We return r10 as a favor to the EPC + * syscall code so that it can quicky determine if the syscall + * needs to be restarted or not. + */ + return (tf->tf_scratch.gr10); +} + +/* * Trap is called from exception.s to handle most types of processor traps. - * System calls are broken out for efficiency and ASTs are broken out - * to make the code a bit cleaner and more representative of the - * architecture. */ /*ARGSUSED*/ void -trap(int vector, int imm, struct trapframe *framep) +trap(int vector, struct trapframe *framep) { - struct thread *td; struct proc *p; - int i; + struct thread *td; u_int64_t ucode; + int i, user; u_int sticks; - int user; - cnt.v_trap++; + user = ((framep->tf_special.psr & IA64_PSR_CPL) == IA64_PSR_CPL_USER); + + /* Short-circuit break instruction based system calls. */ + if (vector == IA64_VEC_BREAK && user && + framep->tf_special.ifa == 0x100000) { + break_syscall(framep); + return; + } + + /* Sanitize the FP state in case the user has trashed it. */ + ia64_set_fpsr(IA64_FPSR_DEFAULT); + + atomic_add_int(&cnt.v_trap, 1); + td = curthread; p = td->td_proc; ucode = 0; - /* - * Make sure we have a sane floating-point state in case the - * user has trashed it. - */ - ia64_set_fpsr(IA64_FPSR_DEFAULT); - - user = ((framep->tf_cr_ipsr & IA64_PSR_CPL) == IA64_PSR_CPL_USER); if (user) { sticks = td->td_sticks; td->td_frame = framep; @@ -341,19 +373,18 @@ trap(int vector, int imm, struct trapframe *framep) } switch (vector) { - case IA64_VEC_UNALIGNED_REFERENCE: + + case IA64_VEC_UNALIGNED_REFERENCE: { /* * If user-land, do whatever fixups, printing, and * signalling is appropriate (based on system-wide * and per-process unaligned-access-handling flags). */ if (user) { - mtx_lock(&Giant); i = unaligned_fixup(framep, td); - mtx_unlock(&Giant); if (i == 0) goto out; - ucode = framep->tf_cr_ifa; /* VA */ + ucode = framep->tf_special.ifa; /* VA */ break; } @@ -369,9 +400,9 @@ trap(int vector, int imm, struct trapframe *framep) * does cause an unaligned access it's a kernel bug. */ goto dopanic; + } - case IA64_VEC_FLOATING_POINT_FAULT: - { + case IA64_VEC_FLOATING_POINT_FAULT: { FP_STATE fp_state; FPSWA_RET fpswa_ret; FPSWA_BUNDLE bundle; @@ -385,7 +416,7 @@ trap(int vector, int imm, struct trapframe *framep) break; } mtx_lock(&Giant); - i = copyin((const void *)(framep->tf_cr_iip), &bundle, 16); + i = copyin((void *)(framep->tf_special.iip), &bundle, 16); mtx_unlock(&Giant); if (i) { i = SIGBUS; /* EFAULT, basically */ @@ -396,28 +427,28 @@ trap(int vector, int imm, struct trapframe *framep) fp_state.bitmask_low64 = 0xffc0; /* bits 6 - 15 */ fp_state.bitmask_high64 = 0x0; fp_state.fp_low_preserved = NULL; - fp_state.fp_low_volatile = framep->tf_f; + fp_state.fp_low_volatile = &framep->tf_scratch_fp.fr6; fp_state.fp_high_preserved = NULL; fp_state.fp_high_volatile = NULL; /* The docs are unclear. Is Fpswa reentrant? */ fpswa_ret = fpswa_interface->Fpswa(1, &bundle, - &framep->tf_cr_ipsr, &framep->tf_ar_fpsr, - &framep->tf_cr_isr, &framep->tf_pr, - &framep->tf_cr_ifs, &fp_state); + &framep->tf_special.psr, &framep->tf_special.fpsr, + &framep->tf_special.isr, &framep->tf_special.pr, + &framep->tf_special.cfm, &fp_state); if (fpswa_ret.status == 0) { /* fixed. update ipsr and iip to next insn */ int ei; - ei = (framep->tf_cr_isr >> 41) & 0x03; + ei = (framep->tf_special.isr >> 41) & 0x03; if (ei == 0) { /* no template for this case */ - framep->tf_cr_ipsr &= ~IA64_ISR_EI; - framep->tf_cr_ipsr |= IA64_ISR_EI_1; + framep->tf_special.psr &= ~IA64_ISR_EI; + framep->tf_special.psr |= IA64_ISR_EI_1; } else if (ei == 1) { /* MFI or MFB */ - framep->tf_cr_ipsr &= ~IA64_ISR_EI; - framep->tf_cr_ipsr |= IA64_ISR_EI_2; + framep->tf_special.psr &= ~IA64_ISR_EI; + framep->tf_special.psr |= IA64_ISR_EI_2; } else if (ei == 2) { /* MMF */ - framep->tf_cr_ipsr &= ~IA64_ISR_EI; - framep->tf_cr_iip += 0x10; + framep->tf_special.psr &= ~IA64_ISR_EI; + framep->tf_special.iip += 0x10; } goto out; } else if (fpswa_ret.status == -1) { @@ -446,8 +477,7 @@ trap(int vector, int imm, struct trapframe *framep) } } - case IA64_VEC_FLOATING_POINT_TRAP: - { + case IA64_VEC_FLOATING_POINT_TRAP: { FP_STATE fp_state; FPSWA_RET fpswa_ret; FPSWA_BUNDLE bundle; @@ -461,7 +491,7 @@ trap(int vector, int imm, struct trapframe *framep) break; } mtx_lock(&Giant); - i = copyin((const void *)(framep->tf_cr_iip), &bundle, 16); + i = copyin((void *)(framep->tf_special.iip), &bundle, 16); mtx_unlock(&Giant); if (i) { i = SIGBUS; /* EFAULT, basically */ @@ -472,14 +502,14 @@ trap(int vector, int imm, struct trapframe *framep) fp_state.bitmask_low64 = 0xffc0; /* bits 6 - 15 */ fp_state.bitmask_high64 = 0x0; fp_state.fp_low_preserved = NULL; - fp_state.fp_low_volatile = framep->tf_f; + fp_state.fp_low_volatile = &framep->tf_scratch_fp.fr6; fp_state.fp_high_preserved = NULL; fp_state.fp_high_volatile = NULL; /* The docs are unclear. Is Fpswa reentrant? */ fpswa_ret = fpswa_interface->Fpswa(0, &bundle, - &framep->tf_cr_ipsr, &framep->tf_ar_fpsr, - &framep->tf_cr_isr, &framep->tf_pr, - &framep->tf_cr_ifs, &fp_state); + &framep->tf_special.psr, &framep->tf_special.fpsr, + &framep->tf_special.isr, &framep->tf_special.pr, + &framep->tf_special.cfm, &fp_state); if (fpswa_ret.status == 0) { /* fixed */ /* @@ -501,24 +531,97 @@ trap(int vector, int imm, struct trapframe *framep) } } - case IA64_VEC_DISABLED_FP: + case IA64_VEC_DISABLED_FP: { /* High FP registers are disabled. */ + struct pcpu *pcpu; + struct pcb *pcb; + struct thread *thr; + + /* Always fatal in kernel. Should never happen. */ + if (!user) + goto dopanic; + + pcb = td->td_pcb; + pcpu = pcb->pcb_fpcpu; + +#if 0 + printf("XXX: td %p: highfp on cpu %p\n", td, pcpu); +#endif + /* - * on exit from the kernel, if thread == fpcurthread, - * FP is enabled. + * The pcpu variable holds the address of the per-CPU + * structure of the CPU currently holding this threads + * high FP registers (or NULL if no CPU holds these + * registers). We have to interrupt that CPU and wait + * for it to have saved the registers. */ - if (PCPU_GET(fpcurthread) == td) { - printf("trap: fp disabled for fpcurthread == %p", td); - goto dopanic; + if (pcpu != NULL) { + thr = pcpu->pc_fpcurthread; + KASSERT(thr == td, ("High FP state out of sync")); + + if (pcpu == pcpup) { + /* + * Short-circuit handling the trap when this + * CPU already holds the high FP registers for + * this thread. We really shouldn't get the + * trap in the first place, but since it's + * only a performance issue and not a + * correctness issue, we emit a message for + * now, enable the high FP registers and + * return. + */ + printf("XXX: bogusly disabled high FP regs\n"); + framep->tf_special.psr &= ~IA64_PSR_DFH; + goto out; + } +#ifdef SMP + /* + * Interrupt the other CPU so that it saves the high + * FP registers of this thread. Note that this can + * only happen for the SMP case. + */ + ipi_send(pcpu->pc_lid, IPI_HIGH_FP); +#endif +#ifdef DIAGNOSTICS + } else { + KASSERT(PCPU_GET(fpcurthread) != td, + ("High FP state out of sync")); +#endif } - - ia64_fpstate_switch(td); + + thr = PCPU_GET(fpcurthread); + +#if 0 + printf("XXX: cpu %p: highfp belongs to td %p\n", pcpup, thr); +#endif + + /* + * The thr variable holds the thread that owns the high FP + * registers currently on this CPU. Free this CPU so that + * we can load the current threads high FP registers. + */ + if (thr != NULL) { + KASSERT(thr != td, ("High FP state out of sync")); + pcb = thr->td_pcb; + KASSERT(pcb->pcb_fpcpu == pcpup, + ("High FP state out of sync")); + ia64_highfp_save(thr); + } + + /* + * Wait for the other CPU to have saved out high FP + * registers (if applicable). + */ + while (pcpu && pcpu->pc_fpcurthread == td); + + ia64_highfp_load(td); + framep->tf_special.psr &= ~IA64_PSR_DFH; goto out; break; + } case IA64_VEC_PAGE_NOT_PRESENT: case IA64_VEC_INST_ACCESS_RIGHTS: - case IA64_VEC_DATA_ACCESS_RIGHTS: - { + case IA64_VEC_DATA_ACCESS_RIGHTS: { vm_offset_t va; struct vmspace *vm; vm_map_t map; @@ -526,7 +629,7 @@ trap(int vector, int imm, struct trapframe *framep) int rv; rv = 0; - va = framep->tf_cr_ifa; + va = framep->tf_special.ifa; /* * If it was caused by fuswintr or suswintr, just punt. Note @@ -536,8 +639,8 @@ trap(int vector, int imm, struct trapframe *framep) */ if (!user && td != NULL && td->td_pcb->pcb_accessaddr == va && td->td_pcb->pcb_onfault == (unsigned long)fswintrberr) { - framep->tf_cr_iip = td->td_pcb->pcb_onfault; - framep->tf_cr_ipsr &= ~IA64_PSR_RI; + framep->tf_special.iip = td->td_pcb->pcb_onfault; + framep->tf_special.psr &= ~IA64_PSR_RI; td->td_pcb->pcb_onfault = 0; goto out; } @@ -559,9 +662,9 @@ trap(int vector, int imm, struct trapframe *framep) map = &vm->vm_map; } - if (framep->tf_cr_isr & IA64_ISR_X) + if (framep->tf_special.isr & IA64_ISR_X) ftype = VM_PROT_EXECUTE; - else if (framep->tf_cr_isr & IA64_ISR_W) + else if (framep->tf_special.isr & IA64_ISR_W) ftype = VM_PROT_WRITE; else ftype = VM_PROT_READ; @@ -597,8 +700,9 @@ trap(int vector, int imm, struct trapframe *framep) if (!user) { /* Check for copyin/copyout fault. */ if (td != NULL && td->td_pcb->pcb_onfault != 0) { - framep->tf_cr_iip = td->td_pcb->pcb_onfault; - framep->tf_cr_ipsr &= ~IA64_PSR_RI; + framep->tf_special.iip = + td->td_pcb->pcb_onfault; + framep->tf_special.psr &= ~IA64_PSR_RI; td->td_pcb->pcb_onfault = 0; goto out; } @@ -609,10 +713,10 @@ trap(int vector, int imm, struct trapframe *framep) break; } - case IA64_VEC_SINGLE_STEP_TRAP: - case IA64_VEC_DEBUG: - case IA64_VEC_TAKEN_BRANCH_TRAP: case IA64_VEC_BREAK: + case IA64_VEC_DEBUG: + case IA64_VEC_SINGLE_STEP_TRAP: + case IA64_VEC_TAKEN_BRANCH_TRAP: { /* * These are always fatal in kernel, and should never happen. */ @@ -633,27 +737,29 @@ trap(int vector, int imm, struct trapframe *framep) } i = SIGTRAP; break; + } - case IA64_VEC_GENERAL_EXCEPTION: + case IA64_VEC_GENERAL_EXCEPTION: { if (user) { ucode = vector; i = SIGILL; break; } goto dopanic; + } case IA64_VEC_UNSUPP_DATA_REFERENCE: - case IA64_VEC_LOWER_PRIVILEGE_TRANSFER: + case IA64_VEC_LOWER_PRIVILEGE_TRANSFER: { if (user) { ucode = vector; i = SIGBUS; break; } goto dopanic; + } - case IA64_VEC_IA32_EXCEPTION: - { - u_int64_t isr = framep->tf_cr_isr; + case IA64_VEC_IA32_EXCEPTION: { + u_int64_t isr = framep->tf_special.isr; switch ((isr >> 16) & 0xffff) { case IA32_EXCEPTION_DIVIDE: @@ -694,7 +800,7 @@ trap(int vector, int imm, struct trapframe *framep) break; case IA32_EXCEPTION_ALIGNMENT_CHECK: - ucode = framep->tf_cr_ifa; /* VA */ + ucode = framep->tf_special.ifa; /* VA */ i = SIGBUS; break; @@ -709,33 +815,36 @@ trap(int vector, int imm, struct trapframe *framep) break; } - case IA64_VEC_IA32_INTERRUPT: + case IA64_VEC_IA32_INTERRUPT: { /* * INT n instruction - probably a syscall. */ - if (((framep->tf_cr_isr >> 16) & 0xffff) == 0x80) { + if (((framep->tf_special.isr >> 16) & 0xffff) == 0x80) { ia32_syscall(framep); goto out; } else { - ucode = (framep->tf_cr_isr >> 16) & 0xffff; + ucode = (framep->tf_special.isr >> 16) & 0xffff; i = SIGILL; break; } + } - case IA64_VEC_IA32_INTERCEPT: + case IA64_VEC_IA32_INTERCEPT: { /* * Maybe need to emulate ia32 instruction. */ goto dopanic; + } default: goto dopanic; } -#ifdef DEBUG - printtrap(vector, imm, framep, 1, user); -#endif + if (print_usertrap) + printtrap(vector, framep, 1, user); + trapsignal(td, i, ucode); + out: if (user) { userret(td, framep, sticks); @@ -743,73 +852,96 @@ out: #ifdef DIAGNOSTIC cred_free_thread(td); #endif + do_ast(framep); } return; dopanic: - printtrap(vector, imm, framep, 1, user); - - /* XXX dump registers */ - + printtrap(vector, framep, 1, user); #ifdef DDB kdb_trap(vector, framep); #endif - panic("trap"); } + +/* + * Handle break instruction based system calls. + */ +void +break_syscall(struct trapframe *tf) +{ + uint64_t *bsp, *tfp; + uint64_t iip, psr; + int error, nargs; + + /* Save address of break instruction. */ + iip = tf->tf_special.iip; + psr = tf->tf_special.psr; + + /* Advance to the next instruction. */ + tf->tf_special.psr += IA64_PSR_RI_1; + if ((tf->tf_special.psr & IA64_PSR_RI) > IA64_PSR_RI_2) { + tf->tf_special.iip += 16; + tf->tf_special.psr &= ~IA64_PSR_RI; + } + + /* + * Copy the arguments on the register stack into the trapframe + * to avoid having interleaved NaT collections. + */ + tfp = &tf->tf_scratch.gr16; + nargs = tf->tf_special.cfm & 0x7f; + bsp = (uint64_t*)(curthread->td_kstack + tf->tf_special.ndirty); + bsp -= (((uintptr_t)bsp & 0x1ff) < (nargs << 3)) ? (nargs + 1): nargs; + while (nargs--) { + *tfp++ = *bsp++; + if (((uintptr_t)bsp & 0x1ff) == 0x1f8) + bsp++; + } + error = syscall(tf); + if (error == ERESTART) { + tf->tf_special.iip = iip; + tf->tf_special.psr = psr; + } + + do_ast(tf); +} + /* * Process a system call. * - * System calls are strange beasts. They are passed the syscall number - * in r15, and the arguments in the registers (as normal). They return - * an error flag in r10 (if r10 != 0 on return, the syscall had an error), - * and the return value (if any) in r8 and r9. - * - * The assembly stub takes care of moving the call number into a register - * we can get to, and moves all of the argument registers into a stack - * buffer. On return, it restores r8-r10 from the frame before - * returning to the user process. + * See syscall.s for details as to how we get here. In order to support + * the ERESTART case, we return the error to our caller. They deal with + * the hairy details. */ -void -syscall(int code, u_int64_t *args, struct trapframe *framep) +int +syscall(struct trapframe *tf) { struct sysent *callp; - struct thread *td; struct proc *p; - int error = 0; - u_int64_t oldip, oldri; + struct thread *td; + u_int64_t *args; + int code, error; u_int sticks; - cnt.v_syscall++; + code = tf->tf_scratch.gr15; + args = &tf->tf_scratch.gr16; + + atomic_add_int(&cnt.v_syscall, 1); + td = curthread; p = td->td_proc; - td->td_frame = framep; + td->td_frame = tf; sticks = td->td_sticks; if (td->td_ucred != p->p_ucred) cred_update_thread(td); - - /* - * Skip past the break instruction. Remember old address in case - * we have to restart. - */ - oldip = framep->tf_cr_iip; - oldri = framep->tf_cr_ipsr & IA64_PSR_RI; - framep->tf_cr_ipsr += IA64_PSR_RI_1; - if ((framep->tf_cr_ipsr & IA64_PSR_RI) > IA64_PSR_RI_2) { - framep->tf_cr_ipsr &= ~IA64_PSR_RI; - framep->tf_cr_iip += 16; - } - if (p->p_flag & P_THREADED) thread_user_enter(p, td); -#ifdef DIAGNOSTIC - ia64_fpstate_check(td); -#endif if (p->p_sysent->sv_prepsyscall) { - /* (*p->p_sysent->sv_prepsyscall)(framep, args, &code, ¶ms); */ + /* (*p->p_sysent->sv_prepsyscall)(tf, args, &code, ¶ms); */ panic("prepsyscall"); } else { /* @@ -842,39 +974,33 @@ syscall(int code, u_int64_t *args, struct trapframe *framep) if (KTRPOINT(td, KTR_SYSCALL)) ktrsyscall(code, (callp->sy_narg & SYF_ARGMASK), args); #endif - if (error == 0) { - td->td_retval[0] = 0; - td->td_retval[1] = 0; - STOPEVENT(p, S_SCE, (callp->sy_narg & SYF_ARGMASK)); + td->td_retval[0] = 0; + td->td_retval[1] = 0; + tf->tf_scratch.gr10 = EJUSTRETURN; - error = (*callp->sy_call)(td, args); - } + STOPEVENT(p, S_SCE, (callp->sy_narg & SYF_ARGMASK)); + error = (*callp->sy_call)(td, args); - switch (error) { - case 0: - framep->tf_r[FRAME_R8] = td->td_retval[0]; - framep->tf_r[FRAME_R9] = td->td_retval[1]; - framep->tf_r[FRAME_R10] = 0; - break; - case ERESTART: - framep->tf_cr_iip = oldip; - framep->tf_cr_ipsr = - (framep->tf_cr_ipsr & ~IA64_PSR_RI) | oldri; - break; - case EJUSTRETURN: - break; - default: - if (p->p_sysent->sv_errsize) { - if (error >= p->p_sysent->sv_errsize) - error = -1; /* XXX */ - else + if (error != EJUSTRETURN) { + /* + * Save the "raw" error code in r10. We use this to handle + * syscall restarts (see do_ast()). + */ + tf->tf_scratch.gr10 = error; + if (error == 0) { + tf->tf_scratch.gr8 = td->td_retval[0]; + tf->tf_scratch.gr9 = td->td_retval[1]; + } else if (error != ERESTART) { + if (error < p->p_sysent->sv_errsize) error = p->p_sysent->sv_errtbl[error]; + /* + * Translated error codes are returned in r8. User + * processes use the translated error code. + */ + tf->tf_scratch.gr8 = error; } - framep->tf_r[FRAME_R8] = error; - framep->tf_r[FRAME_R10] = 1; - break; } /* @@ -883,12 +1009,13 @@ syscall(int code, u_int64_t *args, struct trapframe *framep) if ((callp->sy_narg & SYF_MPSAFE) == 0) mtx_unlock(&Giant); - userret(td, framep, sticks); + userret(td, tf, sticks); #ifdef KTRACE if (KTRPOINT(td, KTR_SYSRET)) ktrsysret(code, error, td->td_retval[0]); #endif + /* * This works because errno is findable through the * register set. If we ever support an emulation where this @@ -899,10 +1026,13 @@ syscall(int code, u_int64_t *args, struct trapframe *framep) #ifdef DIAGNOSTIC cred_free_thread(td); #endif + WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning", (code >= 0 && code < SYS_MAXSYSCALL) ? syscallnames[code] : "???"); mtx_assert(&sched_lock, MA_NOTOWNED); mtx_assert(&Giant, MA_NOTOWNED); + + return (error); } #include <i386/include/psl.h> @@ -933,9 +1063,9 @@ ia32_syscall(struct trapframe *framep) td->td_frame = framep; if (td->td_ucred != p->p_ucred) cred_update_thread(td); - params = (caddr_t)(framep->tf_r[FRAME_SP] & ((1L<<32)-1)) - + sizeof(u_int32_t); - code = framep->tf_r[FRAME_R8]; /* eax */ + params = (caddr_t)(framep->tf_special.sp & ((1L<<32)-1)) + + sizeof(u_int32_t); + code = framep->tf_scratch.gr8; /* eax */ orig_eflags = ia64_get_eflag(); if (p->p_sysent->sv_prepsyscall) { @@ -1001,7 +1131,7 @@ ia32_syscall(struct trapframe *framep) if (error == 0) { td->td_retval[0] = 0; - td->td_retval[1] = framep->tf_r[FRAME_R10]; /* edx */ + td->td_retval[1] = framep->tf_scratch.gr10; /* edx */ STOPEVENT(p, S_SCE, narg); @@ -1010,8 +1140,8 @@ ia32_syscall(struct trapframe *framep) switch (error) { case 0: - framep->tf_r[FRAME_R8] = td->td_retval[0]; /* eax */ - framep->tf_r[FRAME_R10] = td->td_retval[1]; /* edx */ + framep->tf_scratch.gr8 = td->td_retval[0]; /* eax */ + framep->tf_scratch.gr10 = td->td_retval[1]; /* edx */ ia64_set_eflag(ia64_get_eflag() & ~PSL_C); break; @@ -1020,7 +1150,7 @@ ia32_syscall(struct trapframe *framep) * Reconstruct pc, assuming lcall $X,y is 7 bytes, * int 0x80 is 2 bytes. XXX Assume int 0x80. */ - framep->tf_cr_iip -= 2; + framep->tf_special.iip -= 2; break; case EJUSTRETURN: @@ -1033,7 +1163,7 @@ ia32_syscall(struct trapframe *framep) else error = p->p_sysent->sv_errtbl[error]; } - framep->tf_r[FRAME_R8] = error; + framep->tf_scratch.gr8 = error; ia64_set_eflag(ia64_get_eflag() | PSL_C); break; } diff --git a/sys/ia64/ia64/unaligned.c b/sys/ia64/ia64/unaligned.c index 78981be..0ebc2db 100644 --- a/sys/ia64/ia64/unaligned.c +++ b/sys/ia64/ia64/unaligned.c @@ -34,7 +34,6 @@ #include <vm/vm_extern.h> #include <machine/frame.h> #include <machine/inst.h> -#include <machine/rse.h> #define sign_extend(imm, w) (((int64_t)(imm) << (64 - (w))) >> (64 - (w))) @@ -153,16 +152,45 @@ static int read_register(struct trapframe *framep, struct thread *td, int reg, u_int64_t *valuep) { - if (reg == 0) { - *valuep = 0; - return 0; - } else if (reg < 32) { - *valuep = framep->tf_r[reg - 1]; - return 0; + + if (reg < 32) { + switch (reg) { + case 0: *valuep = 0; break; + case 1: *valuep = framep->tf_special.gp; break; + case 2: *valuep = framep->tf_scratch.gr2; break; + case 3: *valuep = framep->tf_scratch.gr3; break; + case 8: *valuep = framep->tf_scratch.gr8; break; + case 9: *valuep = framep->tf_scratch.gr9; break; + case 10: *valuep = framep->tf_scratch.gr10; break; + case 11: *valuep = framep->tf_scratch.gr11; break; + case 12: *valuep = framep->tf_special.sp; break; + case 13: *valuep = framep->tf_special.tp; break; + case 14: *valuep = framep->tf_scratch.gr14; break; + case 15: *valuep = framep->tf_scratch.gr15; break; + case 16: *valuep = framep->tf_scratch.gr16; break; + case 17: *valuep = framep->tf_scratch.gr17; break; + case 18: *valuep = framep->tf_scratch.gr18; break; + case 19: *valuep = framep->tf_scratch.gr19; break; + case 20: *valuep = framep->tf_scratch.gr20; break; + case 21: *valuep = framep->tf_scratch.gr21; break; + case 22: *valuep = framep->tf_scratch.gr22; break; + case 23: *valuep = framep->tf_scratch.gr23; break; + case 24: *valuep = framep->tf_scratch.gr24; break; + case 25: *valuep = framep->tf_scratch.gr25; break; + case 26: *valuep = framep->tf_scratch.gr26; break; + case 27: *valuep = framep->tf_scratch.gr27; break; + case 28: *valuep = framep->tf_scratch.gr28; break; + case 29: *valuep = framep->tf_scratch.gr29; break; + case 30: *valuep = framep->tf_scratch.gr30; break; + case 31: *valuep = framep->tf_scratch.gr31; break; + default: + return (EINVAL); + } } else { - u_int64_t cfm = framep->tf_cr_ifs; - u_int64_t *bsp = (u_int64_t *) (td->td_kstack - + framep->tf_ndirty); +#if 0 + u_int64_t cfm = framep->tf_special.cfm; + u_int64_t *bsp = (u_int64_t *)(td->td_kstack + + framep->tf_ndirty); int sof = cfm & 0x7f; int sor = 8*((cfm >> 14) & 15); int rrb_gr = (cfm >> 18) & 0x7f; @@ -182,23 +210,54 @@ read_register(struct trapframe *framep, struct thread *td, } *valuep = *ia64_rse_register_address(bsp, reg); - return 0; + return (0); +#else + return (EINVAL); +#endif } - - return EINVAL; + return (0); } static int write_register(struct trapframe *framep, struct thread *td, int reg, u_int64_t value) { - if (reg == 0) { - return EINVAL; /* can't happen */ - } else if (reg < 32) { - framep->tf_r[reg - 1] = value; - return 0; + + if (reg < 32) { + switch (reg) { + case 1: framep->tf_special.gp = value; break; + case 2: framep->tf_scratch.gr2 = value; break; + case 3: framep->tf_scratch.gr3 = value; break; + case 8: framep->tf_scratch.gr8 = value; break; + case 9: framep->tf_scratch.gr9 = value; break; + case 10: framep->tf_scratch.gr10 = value; break; + case 11: framep->tf_scratch.gr11 = value; break; + case 12: framep->tf_special.sp = value; break; + case 13: framep->tf_special.tp = value; break; + case 14: framep->tf_scratch.gr14 = value; break; + case 15: framep->tf_scratch.gr15 = value; break; + case 16: framep->tf_scratch.gr16 = value; break; + case 17: framep->tf_scratch.gr17 = value; break; + case 18: framep->tf_scratch.gr18 = value; break; + case 19: framep->tf_scratch.gr19 = value; break; + case 20: framep->tf_scratch.gr20 = value; break; + case 21: framep->tf_scratch.gr21 = value; break; + case 22: framep->tf_scratch.gr22 = value; break; + case 23: framep->tf_scratch.gr23 = value; break; + case 24: framep->tf_scratch.gr24 = value; break; + case 25: framep->tf_scratch.gr25 = value; break; + case 26: framep->tf_scratch.gr26 = value; break; + case 27: framep->tf_scratch.gr27 = value; break; + case 28: framep->tf_scratch.gr28 = value; break; + case 29: framep->tf_scratch.gr29 = value; break; + case 30: framep->tf_scratch.gr30 = value; break; + case 31: framep->tf_scratch.gr31 = value; break; + default: + return (EINVAL); + } } else { - u_int64_t cfm = framep->tf_cr_ifs; +#if 0 + u_int64_t cfm = framep->tf_special.cfm; u_int64_t *bsp = (u_int64_t *) (td->td_kstack + framep->tf_ndirty); int sof = cfm & 0x7f; @@ -221,9 +280,11 @@ write_register(struct trapframe *framep, struct thread *td, *ia64_rse_register_address(bsp, reg) = value; return 0; +#else + return (EINVAL); +#endif } - - return EINVAL; + return (0); } /* @@ -367,7 +428,7 @@ invala_e(int reg) int unaligned_fixup(struct trapframe *framep, struct thread *td) { - vm_offset_t va = framep->tf_cr_ifa; + vm_offset_t va = framep->tf_special.ifa; int doprint, dofix, dosigbus; int signal, size = 0; unsigned long uac; @@ -399,7 +460,7 @@ unaligned_fixup(struct trapframe *framep, struct thread *td) * If psr.ac is set, then clearly the user program *wants* to * fault. */ - if (framep->tf_cr_ipsr & IA64_PSR_AC) { + if (framep->tf_special.psr & IA64_PSR_AC) { dofix = 0; dosigbus = 1; } @@ -419,10 +480,10 @@ unaligned_fixup(struct trapframe *framep, struct thread *td) * offending instruction. * XXX assume that the instruction is in an 'M' slot. */ - copyin((const void *) framep->tf_cr_iip, &low, 8); - copyin((const void *) (framep->tf_cr_iip + 8), &high, 8); + copyin((const void *) framep->tf_special.iip, &low, 8); + copyin((const void *) (framep->tf_special.iip + 8), &high, 8); ia64_unpack_bundle(low, high, &b); - slot = (framep->tf_cr_ipsr >> 41) & 3; + slot = (framep->tf_special.psr >> 41) & 3; ins.ins = b.slot[slot]; decoded = 0; @@ -451,7 +512,7 @@ unaligned_fixup(struct trapframe *framep, struct thread *td) */ if (doprint) { uprintf("pid %d (%s): unaligned access: va=0x%lx pc=0x%lx", - p->p_pid, p->p_comm, va, framep->tf_cr_iip); + p->p_pid, p->p_comm, va, framep->tf_special.iip); if (decoded) { uprintf(" op="); if (dec.isload) { @@ -500,7 +561,7 @@ unaligned_fixup(struct trapframe *framep, struct thread *td) */ __asm __volatile("flushrs"); - isr = framep->tf_cr_isr; + isr = framep->tf_special.isr; error = read_register(framep, td, dec.basereg, &addr); if (error) { signal = SIGBUS; @@ -565,12 +626,12 @@ unaligned_fixup(struct trapframe *framep, struct thread *td) * Advance to the instruction following the * one which faulted. */ - if ((framep->tf_cr_ipsr & IA64_PSR_RI) + if ((framep->tf_special.psr & IA64_PSR_RI) == IA64_PSR_RI_2) { - framep->tf_cr_ipsr &= ~IA64_PSR_RI; - framep->tf_cr_iip += 16; + framep->tf_special.psr &= ~IA64_PSR_RI; + framep->tf_special.iip += 16; } else { - framep->tf_cr_ipsr += IA64_PSR_RI_1; + framep->tf_special.psr += IA64_PSR_RI_1; } } } else { diff --git a/sys/ia64/ia64/unwind.c b/sys/ia64/ia64/unwind.c index 2da7041..5f1c3e3 100644 --- a/sys/ia64/ia64/unwind.c +++ b/sys/ia64/ia64/unwind.c @@ -34,1479 +34,274 @@ #include <machine/frame.h> #include <machine/unwind.h> -#include <machine/rse.h> -#ifdef UNWIND_DEBUG -#define DPF(x) printf x -#else -#define DPF(x) -#endif - -MALLOC_DEFINE(M_UNWIND, "Unwind table", "Unwind table information"); +#include <uwx.h> -struct ia64_unwind_table_entry { - u_int64_t ue_start; /* procedure start */ - u_int64_t ue_end; /* procedure end */ - u_int64_t ue_info; /* offset to procedure descriptors */ -}; +MALLOC_DEFINE(M_UNWIND, "Unwind", "Unwind information"); -struct ia64_unwind_info { - u_int64_t ui_length : 32; /* length in 64bit units */ - u_int64_t ui_flags : 16; - u_int64_t ui_version : 16; +struct unw_entry { + uint64_t ue_start; /* procedure start */ + uint64_t ue_end; /* procedure end */ + uint64_t ue_info; /* offset to procedure descriptors */ }; -LIST_HEAD(ia64_unwind_table_list, ia64_unwind_table); - -struct ia64_unwind_table { - LIST_ENTRY(ia64_unwind_table) ut_link; - u_int64_t ut_base; - u_int64_t ut_limit; - struct ia64_unwind_table_entry *ut_start; - struct ia64_unwind_table_entry *ut_end; +struct unw_table { + LIST_ENTRY(unw_table) ut_link; + uint64_t ut_base; + uint64_t ut_limit; + struct unw_entry *ut_start; + struct unw_entry *ut_end; }; -struct unwind_reg { - u_int64_t ur_value; /* current value */ - u_int64_t *ur_save; /* save location */ - int ur_when; /* when the save happened */ -}; - -struct unwind_fpreg { - struct ia64_fpreg ur_value; /* current value */ - struct ia64_fpreg *ur_save; /* save location */ - int ur_when; /* when the save happened */ -}; +LIST_HEAD(unw_table_list, unw_table); -struct ia64_unwind_state { - LIST_ENTRY(ia64_unwind_state) us_link; /* free list */ +static struct unw_table_list unw_tables; - /* - * Current register state and location of saved register state - */ - struct register_state { - struct unwind_reg rs_psp; - struct unwind_reg rs_pfs; - struct unwind_reg rs_preds; - struct unwind_reg rs_unat; - struct unwind_reg rs_lc; - struct unwind_reg rs_rnat; - struct unwind_reg rs_bsp; - struct unwind_reg rs_bspstore; - struct unwind_reg rs_fpsr; - struct unwind_reg rs_priunat; - struct unwind_reg rs_br[8]; - struct unwind_reg rs_gr[32]; - struct unwind_fpreg rs_fr[32]; - u_int64_t rs_stack_size; - } us_regs; - - /* - * Variables used while parsing unwind records. - */ - u_int64_t us_ip; /* value of IP for this frame */ - int us_ri; /* RI field from cr.ipsr */ - u_int64_t us_pc; /* slot offset in procedure */ - u_int64_t *us_bsp; /* backing store for frame */ - u_int64_t us_cfm; /* CFM value for frame */ - u_int64_t *us_spill; /* spill_base location */ - int us_spilloff; /* offset into spill area */ - int us_grmask; /* mask of grs being spilled */ - int us_frmask; /* mask of frs being spilled */ - int us_brmask; /* mask of brs being spilled */ -}; - -static int ia64_unwind_initialised; -static struct ia64_unwind_table_list ia64_unwind_tables; -#define MAX_UNWIND_STATES 4 -static struct ia64_unwind_state ia64_unwind_state_static[MAX_UNWIND_STATES]; -static LIST_HEAD(ia64_unwind_state_list, ia64_unwind_state) ia64_unwind_states; - -static void -ia64_initialise_unwind(void *arg __unused) +static void * +unw_alloc(size_t sz) { - int i; - - KASSERT(!ia64_unwind_initialised, ("foo")); - LIST_INIT(&ia64_unwind_tables); - LIST_INIT(&ia64_unwind_states); - for (i = 0; i < MAX_UNWIND_STATES; i++) { - LIST_INSERT_HEAD(&ia64_unwind_states, - &ia64_unwind_state_static[i], us_link); - } - - ia64_unwind_initialised = 1; + return (malloc(sz, M_UNWIND, M_WAITOK)); } -SYSINIT(unwind, SI_SUB_KMEM, SI_ORDER_ANY, ia64_initialise_unwind, 0); -static struct ia64_unwind_table * -find_table(u_int64_t ip) +static void +unw_free(void *p) { - struct ia64_unwind_table *ut; - LIST_FOREACH(ut, &ia64_unwind_tables, ut_link) { - if (ip >= ut->ut_base && ip < ut->ut_limit) - return ut; - } - return 0; + free(p, M_UNWIND); } -static struct ia64_unwind_table_entry * -find_entry(struct ia64_unwind_table *ut, u_int64_t ip) +#if 0 +static struct unw_entry * +unw_entry_lookup(struct unw_table *ut, uint64_t ip) { - struct ia64_unwind_table_entry *start; - struct ia64_unwind_table_entry *end; - struct ia64_unwind_table_entry *mid; + struct unw_entry *end, *mid, *start; ip -= ut->ut_base; start = ut->ut_start; end = ut->ut_end - 1; while (start < end) { - mid = start + (end - start) / 2; - if (ip < mid->ue_start) { - if (end == mid) - break; + mid = start + ((end - start) >> 1); + if (ip < mid->ue_start) end = mid; - } else if (ip >= mid->ue_end) { - if (start == mid) - break; - start = mid; - } else - return mid; - } - - return 0; -} - -int -ia64_add_unwind_table(vm_offset_t base, vm_offset_t start, vm_offset_t end) -{ - struct ia64_unwind_table *ut; - - KASSERT(ia64_unwind_initialised, ("foo")); - - ut = malloc(sizeof(struct ia64_unwind_table), M_UNWIND, M_NOWAIT); - if (ut == NULL) - return (ENOMEM); - - ut->ut_base = base; - ut->ut_start = (struct ia64_unwind_table_entry*)start; - ut->ut_end = (struct ia64_unwind_table_entry*)end; - ut->ut_limit = base + ut->ut_end[-1].ue_end; - LIST_INSERT_HEAD(&ia64_unwind_tables, ut, ut_link); - - if (bootverbose) - printf("UNWIND: table added: base=%lx, start=%lx, end=%lx\n", - base, start, end); - - return (0); -} - -void -ia64_delete_unwind_table(vm_offset_t base) -{ - struct ia64_unwind_table *ut; - - KASSERT(ia64_unwind_initialised, ("foo")); - - ut = find_table(base); - if (ut != NULL) { - LIST_REMOVE(ut, ut_link); - free(ut, M_UNWIND); - if (bootverbose) - printf("UNWIND: table removed: base=%lx\n", base); - } -} - -struct ia64_unwind_state * -ia64_create_unwind_state(struct trapframe *framep) -{ - struct ia64_unwind_state *us; - int i; - - if (!ia64_unwind_initialised) - return 0; - - us = LIST_FIRST(&ia64_unwind_states); - if (us) { - LIST_REMOVE(us, us_link); - } else { - us = malloc(sizeof(struct ia64_unwind_state), - M_UNWIND, M_NOWAIT); - if (!us) - return 0; - } - - bzero(us, sizeof(*us)); - us->us_regs.rs_psp.ur_value = framep->tf_r[FRAME_SP]; - us->us_regs.rs_pfs.ur_value = framep->tf_ar_pfs; - us->us_regs.rs_preds.ur_value = framep->tf_pr; - us->us_regs.rs_unat.ur_value = framep->tf_ar_unat; - us->us_regs.rs_rnat.ur_value = framep->tf_ar_rnat; - us->us_regs.rs_bsp.ur_value = - (u_int64_t) (framep->tf_ar_bspstore + framep->tf_ndirty); - us->us_regs.rs_bspstore.ur_value = framep->tf_ar_bspstore; - us->us_regs.rs_fpsr.ur_value = framep->tf_ar_fpsr; - for (i = 0; i < 8; i++) { - us->us_regs.rs_br[i].ur_value = framep->tf_b[i]; - } - us->us_regs.rs_gr[0].ur_value = 0; - for (i = 1; i < 32; i++) { - us->us_regs.rs_gr[i].ur_value = framep->tf_r[i-1]; - } - for (i = 6; i < 16; i++) { - us->us_regs.rs_fr[i].ur_value = framep->tf_f[i-6]; + else if (ip >= mid->ue_end) + start = mid + 1; + else + break; } - - us->us_ip = framep->tf_cr_iip; - us->us_ri = (framep->tf_cr_ipsr & IA64_PSR_RI) >> 41; - us->us_spill = (u_int64_t *) us->us_regs.rs_gr[12].ur_value; - us->us_cfm = framep->tf_cr_ifs; - us->us_bsp = ia64_rse_previous_frame - ((u_int64_t *) us->us_regs.rs_bsp.ur_value, us->us_cfm & 0x7f); - - return us; -} - -void -ia64_free_unwind_state(struct ia64_unwind_state *us) -{ - LIST_INSERT_HEAD(&ia64_unwind_states, us, us_link); -} - -u_int64_t -ia64_unwind_state_get_ip(struct ia64_unwind_state *us) -{ - return us->us_ip + us->us_ri; -} - -u_int64_t -ia64_unwind_state_get_sp(struct ia64_unwind_state *us) -{ - return us->us_regs.rs_gr[12].ur_value; -} - -u_int64_t -ia64_unwind_state_get_cfm(struct ia64_unwind_state *us) -{ - return us->us_cfm; -} - -u_int64_t * -ia64_unwind_state_get_bsp(struct ia64_unwind_state *us) -{ - return us->us_bsp; -} - -static u_int64_t -read_uleb128(u_int8_t **pp) -{ - u_int8_t *p = *pp; - u_int8_t b; - u_int64_t res; - - res = 0; - do { - b = *p++; - res = (res << 7) | (b & 0x7f); - } while (b & (1 << 7)); - - *pp = p; - return res; -} - -#define PROCESS_WHEN(us, reg, t) \ -do { \ - DPF(("register %s was saved at offset %d\n", \ - #reg, t)); \ - us->us_regs.rs_##reg.ur_when = t; \ -} while (0) - -#define PROCESS_GR(us, reg, gr) \ -do { \ - DPF(("save location for %s at r%d\n", #reg, gr)); \ - us->us_regs.rs_##reg.ur_save = find_gr(us, gr); \ -} while (0) \ - -#define PROCESS_BR(us, reg, br) \ -do { \ - DPF(("save location for %s at b%d\n", #reg, br)); \ - us->us_regs.rs_##reg.ur_save = \ - &us->us_regs.rs_br[br].ur_value; \ -} while (0) - -#define PROCESS_GRMEM(us, reg) \ -do { \ - DPF(("save location for %s at spill+%d\n", \ - #reg, us->us_spilloff)); \ - us->us_regs.rs_##reg.ur_save = \ - &us->us_spill[us->us_spilloff]; \ - us->us_spilloff += 8; \ -} while (0) - -#define PROCESS_FRMEM(us, reg) \ -do { \ - DPF(("save location for %s at spill+%d\n", \ - #reg, us->us_spilloff)); \ - us->us_regs.rs_##reg.ur_save = \ - (struct ia64_fpreg *) \ - &us->us_spill[us->us_spilloff]; \ - us->us_spilloff += 16; \ -} while (0) - -#define PROCESS_SPREL(us, reg, spoff) \ -do { \ - DPF(("save location for %s at sp+%d\n", \ - #reg, 4*spoff)); \ - us->us_regs.rs_##reg.ur_save = (u_int64_t *) \ - (us->us_regs.rs_gr[12].ur_value + 4*spoff); \ -} while (0) - -#define PROCESS_SPREL_WHEN(us, reg, spoff, t) \ -do { \ - PROCESS_SPREL(us, reg, spoff); \ - PROCESS_WHEN(us, reg, t); \ -} while (0) - -#define PROCESS_PSPREL(us, reg, pspoff) \ -do { \ - DPF(("save location for %s at psp+%d\n", \ - #reg, 16-4*pspoff)); \ - us->us_regs.rs_##reg.ur_save = (u_int64_t *) \ - (us->us_regs.rs_psp.ur_value + 16-4*pspoff); \ -} while (0) - -#define PROCESS_PSPREL_WHEN(us, reg, pspoff, t) \ -do { \ - PROCESS_PSPREL(us, reg, pspoff); \ - PROCESS_WHEN(us, reg, t); \ -} while (0) - -static u_int64_t * -find_gr(struct ia64_unwind_state *us, int gr) -{ - if (gr < 32) - return &us->us_regs.rs_gr[gr].ur_value; - else - return ia64_rse_register_address(us->us_bsp, gr); + return ((start < end) ? mid : NULL); } +#endif -static void -parse_prologue(struct ia64_unwind_state *us, int rlen) +static struct unw_table * +unw_table_lookup(uint64_t ip) { -} + struct unw_table *ut; -static void -parse_prologue_gr(struct ia64_unwind_state *us, int rlen, - int mask, int grsave) -{ - if (mask & 8) { - PROCESS_GR(us, br[0], grsave); - grsave++; - } - if (mask & 4) { - PROCESS_GR(us, pfs, grsave); - grsave++; - } - if (mask & 2) { - PROCESS_GR(us, psp, grsave); - grsave++; - } - if (mask & 1) { - PROCESS_GR(us, preds, grsave); - grsave++; + LIST_FOREACH(ut, &unw_tables, ut_link) { + if (ip >= ut->ut_base && ip < ut->ut_limit) + return (ut); } + return (NULL); } -static void -parse_mem_stack_f(struct ia64_unwind_state *us, int t, int size) -{ - DPF(("restore value for psp is sp+%d at offset %d\n", - 16*size, t)); - us->us_regs.rs_psp.ur_when = t; - us->us_regs.rs_stack_size = 16*size; -} - -static void -parse_mem_stack_v(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, psp, t); -} - -static void -parse_psp_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, psp, gr); -} - -static void -parse_psp_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, psp, spoff); -} - -static void -parse_rp_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, br[0], t); -} - -static void -parse_rp_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, br[0], gr); -} - -static void -parse_rp_br(struct ia64_unwind_state *us, int br) -{ - PROCESS_BR(us, br[0], br); -} - -static void -parse_rp_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, br[0], pspoff); -} - -static void -parse_rp_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, br[0], spoff); -} - -static void -parse_pfs_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, pfs, t); -} - -static void -parse_pfs_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, pfs, gr); -} - -static void -parse_pfs_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, pfs, pspoff); -} - -static void -parse_pfs_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, pfs, spoff); -} - -static void -parse_preds_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, preds, t); -} - -static void -parse_preds_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, preds, gr); -} - -static void -parse_preds_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, preds, pspoff); -} - -static void -parse_preds_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, preds, spoff); -} - -static void -parse_fr_mem(struct ia64_unwind_state *us, int frmask) -{ - us->us_frmask = frmask; -} - -static void -parse_frgr_mem(struct ia64_unwind_state *us, int grmask, int frmask) +static int +unw_cb_copyin(int req, char *to, uint64_t from, int len, intptr_t tok) { - us->us_grmask = grmask; - if (grmask & 1) - PROCESS_GRMEM(us, gr[4]); - if (grmask & 2) - PROCESS_GRMEM(us, gr[5]); - if (grmask & 4) - PROCESS_GRMEM(us, gr[6]); - if (grmask & 8) - PROCESS_GRMEM(us, gr[7]); + struct unw_regstate *rs = (void*)tok; + int reg; - us->us_frmask = frmask; - if (frmask & 1) - PROCESS_FRMEM(us, fr[2]); - if (frmask & 2) - PROCESS_FRMEM(us, fr[3]); - if (frmask & 4) - PROCESS_FRMEM(us, fr[4]); - if (frmask & 8) - PROCESS_FRMEM(us, fr[5]); - if (frmask & 16) - PROCESS_FRMEM(us, fr[16]); - if (frmask & 32) - PROCESS_FRMEM(us, fr[17]); - if (frmask & 64) - PROCESS_FRMEM(us, fr[18]); - if (frmask & 128) - PROCESS_FRMEM(us, fr[19]); - if (frmask & 256) - PROCESS_FRMEM(us, fr[20]); - if (frmask & 512) - PROCESS_FRMEM(us, fr[21]); - if (frmask & 1024) - PROCESS_FRMEM(us, fr[22]); - if (frmask & 2048) - PROCESS_FRMEM(us, fr[24]); - if (frmask & 4096) - PROCESS_FRMEM(us, fr[25]); - if (frmask & 8192) - PROCESS_FRMEM(us, fr[26]); - if (frmask & 16384) - PROCESS_FRMEM(us, fr[27]); - if (frmask & 32768) - PROCESS_FRMEM(us, fr[28]); - if (frmask & 65536) - PROCESS_FRMEM(us, fr[29]); - if (frmask & 131072) - PROCESS_FRMEM(us, fr[30]); - if (frmask & 262144) - PROCESS_FRMEM(us, fr[31]); -} + switch (req) { + case UWX_COPYIN_UINFO: + break; + case UWX_COPYIN_MSTACK: + *((uint64_t*)to) = *((uint64_t*)from); + return (8); + case UWX_COPYIN_RSTACK: + *((uint64_t*)to) = *((uint64_t*)from); + return (8); + case UWX_COPYIN_REG: + if (from == UWX_REG_PFS) + from = rs->frame->tf_special.pfs; + else if (from == UWX_REG_PREDS) + from = rs->frame->tf_special.pr; + else if (from == UWX_REG_RNAT) + from = rs->frame->tf_special.rnat; + else if (from == UWX_REG_UNAT) + from = rs->frame->tf_special.unat; + else if (from >= UWX_REG_GR(0) && from <= UWX_REG_GR(127)) { + reg = from - UWX_REG_GR(0); + if (reg == 1) + from = rs->frame->tf_special.gp; + else if (reg == 12) + from = rs->frame->tf_special.sp; + else if (reg == 13) + from = rs->frame->tf_special.tp; + else if (reg >= 2 && reg <= 3) + from = (&rs->frame->tf_scratch.gr2)[reg - 2]; + else if (reg >= 8 && reg <= 11) + from = (&rs->frame->tf_scratch.gr8)[reg - 8]; + else if (reg >= 14 && reg <= 31) + from = (&rs->frame->tf_scratch.gr14)[reg - 14]; + else + goto oops; + } else if (from >= UWX_REG_BR(0) && from <= UWX_REG_BR(7)) { + reg = from - UWX_REG_BR(0); + if (reg == 0) + from = rs->frame->tf_special.rp; + else if (reg >= 6 && reg <= 7) + from = (&rs->frame->tf_scratch.br6)[reg - 6]; + else + goto oops; + } else + goto oops; -static void -parse_gr_gr(struct ia64_unwind_state *us, int grmask, int gr) -{ - us->us_grmask = grmask; - if (grmask & 1) { - PROCESS_GR(us, gr[4], gr); - gr++; + *((uint64_t*)to) = from; + return (len); } - if (grmask & 2) { - PROCESS_GR(us, gr[5], gr); - gr++; - } - if (grmask & 4) { - PROCESS_GR(us, gr[6], gr); - gr++; - } - if (grmask & 8) { - PROCESS_GR(us, gr[7], gr); - gr++; - } -} -static void -parse_gr_mem(struct ia64_unwind_state *us, int grmask) -{ - us->us_grmask = grmask; - if (grmask & 1) - PROCESS_GRMEM(us, gr[4]); - if (grmask & 2) - PROCESS_GRMEM(us, gr[5]); - if (grmask & 4) - PROCESS_GRMEM(us, gr[6]); - if (grmask & 8) - PROCESS_GRMEM(us, gr[7]); -} - -static void -parse_br_mem(struct ia64_unwind_state *us, int brmask) -{ - us->us_brmask = brmask; - if (brmask & 1) - PROCESS_GRMEM(us, br[1]); - if (brmask & 2) - PROCESS_GRMEM(us, br[2]); - if (brmask & 4) - PROCESS_GRMEM(us, br[3]); - if (brmask & 8) - PROCESS_GRMEM(us, br[4]); - if (brmask & 16) - PROCESS_GRMEM(us, br[5]); -} + oops: + printf("UNW: %s(%d, %p, %lx, %d, %lx)\n", __func__, req, to, from, + len, tok); -static void -parse_br_gr(struct ia64_unwind_state *us, int brmask, int gr) -{ - us->us_brmask = brmask; - if (brmask & 1) { - PROCESS_GR(us, br[1], gr); - gr++; - } - if (brmask & 2) { - PROCESS_GR(us, br[2], gr); - gr++; - } - if (brmask & 4) { - PROCESS_GR(us, br[3], gr); - gr++; - } - if (brmask & 8) { - PROCESS_GR(us, br[4], gr); - gr++; - } - if (brmask & 16) { - PROCESS_GR(us, br[5], gr); - gr++; - } -} - -static void -parse_spill_base(struct ia64_unwind_state *us, int pspoff) -{ - DPF(("base of spill area at psp+%d\n", 16 - 4*pspoff)); - us->us_spill = (u_int64_t *) - (us->us_regs.rs_psp.ur_value + 16 - 4*pspoff); + return (0); } -static void -parse_spill_mask(struct ia64_unwind_state *us, int rlen, u_int8_t *imask) -{ - int i, reg; - u_int8_t b; - static int frno[] = { - 2, 3, 4, 5, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 30, 31 - }; - - for (i = 0; i < rlen; i++) { - b = imask[i / 4]; - b = (b >> (2 * (3-(i & 3)))) & 3; - switch (b) { - case 0: - break; - case 1: - reg = frno[ffs(us->us_frmask) - 1]; - DPF(("restoring fr[%d] at offset %d\n", reg, i)); - us->us_regs.rs_fr[reg].ur_when = i; - break; - case 2: - reg = ffs(us->us_grmask) - 1 + 4; - DPF(("restoring gr[%d] at offset %d\n", reg, i)); - us->us_regs.rs_gr[reg].ur_when = i; - break; - case 3: - reg = ffs(us->us_brmask) - 1 + 1; - DPF(("restoring br[%d] at offset %d\n", reg, i)); - us->us_regs.rs_gr[reg].ur_when = i; - break; - } +static int +unw_cb_lookup(int req, uint64_t ip, intptr_t tok, uint64_t **vec) +{ + struct unw_regstate *rs = (void*)tok; + struct unw_table *ut; + + switch (req) { + case UWX_LKUP_LOOKUP: + ut = unw_table_lookup(ip); + if (ut == NULL) + return (UWX_LKUP_NOTFOUND); + rs->keyval[0] = UWX_KEY_TBASE; + rs->keyval[1] = ut->ut_base; + rs->keyval[2] = UWX_KEY_USTART; + rs->keyval[3] = (intptr_t)ut->ut_start; + rs->keyval[4] = UWX_KEY_UEND; + rs->keyval[5] = (intptr_t)ut->ut_end; + rs->keyval[6] = 0; + rs->keyval[7] = 0; + *vec = rs->keyval; + return (UWX_LKUP_UTABLE); + case UWX_LKUP_FREE: + return (0); } -} - -static void -parse_unat_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, unat, t); -} - -static void -parse_unat_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, unat, gr); -} - -static void -parse_unat_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, unat, pspoff); -} - -static void -parse_unat_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, unat, spoff); -} - -static void -parse_lc_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, lc, t); -} - -static void -parse_lc_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, lc, gr); -} - -static void -parse_lc_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, lc, pspoff); -} - -static void -parse_lc_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, lc, spoff); -} - -static void -parse_fpsr_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, fpsr, t); -} - -static void -parse_fpsr_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, fpsr, gr); -} - -static void -parse_fpsr_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, fpsr, pspoff); -} -static void -parse_fpsr_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, fpsr, spoff); -} - -static void -parse_priunat_when_gr(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, priunat, t); + return (UWX_LKUP_ERR); } -static void -parse_priunat_when_mem(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, priunat, t); -} - -static void -parse_priunat_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, priunat, gr); -} - -static void -parse_priunat_psprel(struct ia64_unwind_state *us, int pspoff) +int +unw_create(struct unw_regstate *rs, struct trapframe *tf) { - PROCESS_PSPREL(us, priunat, pspoff); -} + struct unw_table *ut; + uint64_t bsp; + int nats, sof, uwxerr; -static void -parse_priunat_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, priunat, spoff); -} + ut = unw_table_lookup(tf->tf_special.iip); + if (ut == NULL) + return (ENOENT); -static void -parse_bsp_when(struct ia64_unwind_state *us, int t) -{ - PROCESS_WHEN(us, bsp, t); -} + rs->frame = tf; + rs->env = uwx_init(); + if (rs->env == NULL) + return (ENOMEM); -static void -parse_bsp_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, bsp, gr); -} + uwxerr = uwx_register_callbacks(rs->env, (intptr_t)rs, + unw_cb_copyin, unw_cb_lookup); + if (uwxerr) + return (EINVAL); /* XXX */ -static void -parse_bsp_psprel(struct ia64_unwind_state *us, int pspoff) -{ - PROCESS_PSPREL(us, bsp, pspoff); -} + bsp = tf->tf_special.bspstore + tf->tf_special.ndirty; + sof = (int)(tf->tf_special.cfm & 0x7f); + nats = (sof + 63 - ((int)(bsp >> 3) & 0x3f)) / 63; + uwxerr = uwx_init_context(rs->env, tf->tf_special.iip, + tf->tf_special.sp, bsp - ((sof + nats) << 3), tf->tf_special.cfm); -static void -parse_bsp_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, bsp, spoff); + return ((uwxerr) ? EINVAL : 0); /* XXX */ } -static void -parse_bspstore_when(struct ia64_unwind_state *us, int t) +int +unw_step(struct unw_regstate *rs) { - PROCESS_WHEN(us, bspstore, t); -} + int uwxerr; -static void -parse_bspstore_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, bspstore, gr); + uwxerr = uwx_step(rs->env); + return ((uwxerr) ? EINVAL : 0); /* XXX */ } -static void -parse_bspstore_psprel(struct ia64_unwind_state *us, int pspoff) +int +unw_get_bsp(struct unw_regstate *s, uint64_t *r) { - PROCESS_PSPREL(us, bspstore, pspoff); -} + int uwxerr; -static void -parse_bspstore_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, bspstore, spoff); + uwxerr = uwx_get_reg(s->env, UWX_REG_BSP, r); + return ((uwxerr) ? EINVAL : 0); /* XXX */ } -static void -parse_rnat_when(struct ia64_unwind_state *us, int t) +int +unw_get_cfm(struct unw_regstate *s, uint64_t *r) { - PROCESS_WHEN(us, rnat, t); -} + int uwxerr; -static void -parse_rnat_gr(struct ia64_unwind_state *us, int gr) -{ - PROCESS_GR(us, rnat, gr); + uwxerr = uwx_get_reg(s->env, UWX_REG_CFM, r); + return ((uwxerr) ? EINVAL : 0); /* XXX */ } -static void -parse_rnat_psprel(struct ia64_unwind_state *us, int pspoff) +int +unw_get_ip(struct unw_regstate *s, uint64_t *r) { - PROCESS_PSPREL(us, rnat, pspoff); -} + int uwxerr; -static void -parse_rnat_sprel(struct ia64_unwind_state *us, int spoff) -{ - PROCESS_SPREL(us, rnat, spoff); + uwxerr = uwx_get_reg(s->env, UWX_REG_IP, r); + return ((uwxerr) ? EINVAL : 0); /* XXX */ } -static void -parse_epilogue(struct ia64_unwind_state *us, int t, int ecount) +int +unw_table_add(uint64_t base, uint64_t start, uint64_t end) { -} + struct unw_table *ut; -static void -parse_label_state(struct ia64_unwind_state *us, int label) -{ -} + ut = malloc(sizeof(struct unw_table), M_UNWIND, M_NOWAIT); + if (ut == NULL) + return (ENOMEM); -static void -parse_copy_state(struct ia64_unwind_state *us, int label) -{ -} + ut->ut_base = base; + ut->ut_start = (struct unw_entry*)start; + ut->ut_end = (struct unw_entry*)end; + ut->ut_limit = base + ut->ut_end[-1].ue_end; + LIST_INSERT_HEAD(&unw_tables, ut, ut_link); -static void -parse_spill_psprel(struct ia64_unwind_state *us, int t, - int reg, int pspoff) -{ - int type; + if (bootverbose) + printf("UNWIND: table added: base=%lx, start=%lx, end=%lx\n", + base, start, end); - type = reg >> 5; - reg &= 0x1f; - switch (type) { - case 0: - DPF(("save location for gr[%d] at psp+%d at offset %d\n", - reg, 16-4*pspoff, t)); - us->us_regs.rs_gr[reg].ur_save = (u_int64_t *) - (us->us_regs.rs_psp.ur_value + 16-4*pspoff); - us->us_regs.rs_gr[reg].ur_when = t; - break; - case 1: - DPF(("save location for fr[%d] at psp+%d at offset %d\n", - reg, 16-4*pspoff, t)); - us->us_regs.rs_fr[reg].ur_save = (struct ia64_fpreg *) - (us->us_regs.rs_psp.ur_value + 16-4*pspoff); - us->us_regs.rs_fr[reg].ur_when = t; - break; - case 2: - DPF(("save location for br[%d] at psp+%d at offset %d\n", - reg, 16-4*pspoff, t)); - us->us_regs.rs_br[reg].ur_save = (u_int64_t *) - (us->us_regs.rs_psp.ur_value + 16-4*pspoff); - us->us_regs.rs_br[reg].ur_when = t; - break; - case 3: - switch (reg) { - case 0: - PROCESS_PSPREL_WHEN(us, preds, pspoff, t); - break; - case 1: - PROCESS_PSPREL_WHEN(us, psp, pspoff, t); - break; - case 2: - PROCESS_PSPREL_WHEN(us, priunat, pspoff, t); - break; - case 3: - PROCESS_PSPREL_WHEN(us, br[0], pspoff, t); - break; - case 4: - PROCESS_PSPREL_WHEN(us, bsp, pspoff, t); - break; - case 5: - PROCESS_PSPREL_WHEN(us, bspstore, pspoff, t); - break; - case 6: - PROCESS_PSPREL_WHEN(us, rnat, pspoff, t); - break; - case 7: - PROCESS_PSPREL_WHEN(us, unat, pspoff, t); - break; - case 8: - PROCESS_PSPREL_WHEN(us, fpsr, pspoff, t); - break; - case 9: - PROCESS_PSPREL_WHEN(us, pfs, pspoff, t); - break; - case 10: - PROCESS_PSPREL_WHEN(us, lc, pspoff, t); - break; - } - } + return (0); } -static void -parse_spill_sprel(struct ia64_unwind_state *us, int t, - int reg, int spoff) +void +unw_table_remove(uint64_t base) { - int type; + struct unw_table *ut; - type = reg >> 5; - reg &= 0x1f; - switch (type) { - case 0: - DPF(("save location for gr[%d] at sp+%d at offset %d\n", - reg, 4*spoff, t)); - us->us_regs.rs_gr[reg].ur_save = (u_int64_t *) - (us->us_regs.rs_gr[12].ur_value + 4*spoff); - us->us_regs.rs_gr[reg].ur_when = t; - break; - case 1: - DPF(("save location for fr[%d] at sp+%d at offset %d\n", - reg, 4*spoff, t)); - us->us_regs.rs_fr[reg].ur_save = (struct ia64_fpreg *) - (us->us_regs.rs_gr[12].ur_value + 4*spoff); - us->us_regs.rs_fr[reg].ur_when = t; - break; - case 2: - DPF(("save location for br[%d] at sp+%d at offset %d\n", - reg, 4*spoff, t)); - us->us_regs.rs_br[reg].ur_save = (u_int64_t *) - (us->us_regs.rs_gr[12].ur_value + 4*spoff); - us->us_regs.rs_br[reg].ur_when = t; - break; - case 3: - switch (reg) { - case 0: - PROCESS_SPREL_WHEN(us, preds, spoff, t); - break; - case 1: - PROCESS_SPREL_WHEN(us, psp, spoff, t); - break; - case 2: - PROCESS_SPREL_WHEN(us, priunat, spoff, t); - break; - case 3: - PROCESS_SPREL_WHEN(us, br[0], spoff, t); - break; - case 4: - PROCESS_SPREL_WHEN(us, bsp, spoff, t); - break; - case 5: - PROCESS_SPREL_WHEN(us, bspstore, spoff, t); - break; - case 6: - PROCESS_SPREL_WHEN(us, rnat, spoff, t); - break; - case 7: - PROCESS_SPREL_WHEN(us, unat, spoff, t); - break; - case 8: - PROCESS_SPREL_WHEN(us, fpsr, spoff, t); - break; - case 9: - PROCESS_SPREL_WHEN(us, pfs, spoff, t); - break; - case 10: - PROCESS_SPREL_WHEN(us, lc, spoff, t); - break; - } + ut = unw_table_lookup(base); + if (ut != NULL) { + LIST_REMOVE(ut, ut_link); + free(ut, M_UNWIND); + if (bootverbose) + printf("UNWIND: table removed: base=%lx\n", base); } } static void -parse_spill_reg(struct ia64_unwind_state *us, int t, - int reg, int treg) -{ - /* not done yet */ -} - -static void -parse_spill_psprel_p(struct ia64_unwind_state *us, int t, int qp, - int reg, int pspoff) +unw_initialize(void *dummy __unused) { - /* not done yet */ -} - -static void -parse_spill_sprel_p(struct ia64_unwind_state *us, int t, int qp, - int reg, int spoff) -{ - /* not done yet */ -} - -static void -parse_spill_reg_p(struct ia64_unwind_state *us, int t, int qp, - int reg, int treg) -{ - /* not done yet */ -} - -static void -unwind_region(struct register_state *regs, int pc) -{ - int i; - -#define RESTORE(x) \ - do { \ - if (regs->rs_##x.ur_save \ - && pc > regs->rs_##x.ur_when) { \ - DPF(("restoring %s\n", #x)); \ - regs->rs_##x.ur_value = *regs->rs_##x.ur_save; \ - } \ - regs->rs_##x.ur_save = 0; \ - regs->rs_##x.ur_when = 0; \ - } while (0) - -#define RESTORE_INDEX(x, i) \ - do { \ - if (regs->rs_##x[i].ur_save \ - && pc > regs->rs_##x[i].ur_when) { \ - DPF(("restoring %s[%d]\n", #x, i)); \ - regs->rs_##x[i].ur_value = \ - *regs->rs_##x[i].ur_save; \ - } \ - regs->rs_##x[i].ur_save = 0; \ - regs->rs_##x[i].ur_when = 0; \ - } while (0) - - if (regs->rs_stack_size) { - DPF(("restoring psp\n")); - regs->rs_psp.ur_value = - regs->rs_gr[12].ur_value + regs->rs_stack_size; - regs->rs_stack_size = 0; - } else { - RESTORE(psp); - } - RESTORE(pfs); - RESTORE(preds); - RESTORE(unat); - RESTORE(lc); - RESTORE(rnat); - RESTORE(bsp); - RESTORE(bspstore); - RESTORE(fpsr); - RESTORE(priunat); - for (i = 0; i < 8; i++) - RESTORE_INDEX(br, i); - for (i = 0; i < 32; i++) - RESTORE_INDEX(gr, i); - for (i = 0; i < 32; i++) - RESTORE_INDEX(fr, i); - -#undef RESTORE -} - -int -ia64_unwind_state_previous_frame(struct ia64_unwind_state *us) -{ - struct ia64_unwind_table *ut; - struct ia64_unwind_table_entry *ute; - struct ia64_unwind_info *ui; - u_int8_t *p; - u_int8_t *end; - int region = 0; /* 0 for prologue, 1 for body */ - int rlen = 0; - - /* - * Find the entry which describes this procedure. - */ - ut = find_table(us->us_ip); - if (!ut) - return ENOENT; - ute = find_entry(ut, us->us_ip); - if (!ute) { - /* - * If there is no entry for this procedure, we assumes - * its a leaf (i.e. rp and ar.pfs is enough to restore - * the previous frame. - */ - goto noentry; - } - - /* - * Calculate 'pc' as the number of instructions from the start - * of the procedure. - */ - us->us_pc = ((us->us_ip - (ute->ue_start + ut->ut_base)) / 16) * 3 - + us->us_ri; - - /* - * Process unwind records until we find the record which - * contains the pc. - */ - ui = (struct ia64_unwind_info *) (ute->ue_info + ut->ut_base); - p = (u_int8_t *) ui + 8; - end = p + 8 * ui->ui_length; - - while (us->us_pc > 0 && p < end) { - u_int8_t b = *p; - - /* - * Is this a header or a region descriptor? - */ - if ((b >> 7) == 0) { - /* - * Header. - * - * Complete processing of previous region (if - * any) by restoring the appropriate registers - * and ajust pc to be relative to next region. - */ - unwind_region(&us->us_regs, us->us_pc); - us->us_pc -= rlen; - if (us->us_pc <= 0) - break; - - if ((b >> 6) == 0) { - /* R1 */ - region = (b >> 5) & 1; - rlen = b & 0x1f; - parse_prologue(us, rlen); - p++; - } else if ((b >> 3) == 8) { - /* R2 */ - int mask, grsave; - mask = ((b & 7) << 1) | (p[1] >> 7); - grsave = p[1] & 0x7f; - p += 2; - rlen = read_uleb128(&p); - parse_prologue_gr(us, rlen, mask, grsave); - } else if ((b >> 2) == 24) { - /* R3 */ - region = b & 3; - p += 2; - rlen = read_uleb128(&p); - parse_prologue(us, rlen); - } - } else { - if (region == 0) { - /* - * Prologue - */ - if ((b >> 5) == 4) { - /* P1 */ - parse_br_mem(us, b & 0x1f); - p++; - } else if ((b >> 4) == 10) { - /* P2 - br_gr */ - parse_br_gr(us, - (((b & 0xf) << 1) - | (p[1] >> 7)), - p[1] & 0x7f); - p += 2; - } else if ((b >> 3) == 22) { - /* P3 */ - int which, r; - which = ((b & 7) << 1) | (p[1] >> 7); - r = p[1] & 0x7f; - switch (which) { - case 0: - parse_psp_gr(us, r); - break; - case 1: - parse_rp_gr(us, r); - break; - case 2: - parse_pfs_gr(us, r); - break; - case 3: - parse_preds_gr(us, r); - break; - case 4: - parse_unat_gr(us, r); - break; - case 5: - parse_lc_gr(us, r); - break; - case 6: - parse_rp_br(us, r); - break; - case 7: - parse_rnat_gr(us, r); - break; - case 8: - parse_bsp_gr(us, r); - break; - case 9: - parse_bspstore_gr(us, r); - break; - case 10: - parse_fpsr_gr(us, r); - break; - case 11: - parse_priunat_gr(us, r); - break; - } - p += 2; - } else if ((b >> 0) == 184) { - /* P4 */ - parse_spill_mask(us, rlen, p+1); - p += 1 + (rlen + 3) / 4; - } else if ((b >> 0) == 185) { - /* P5 - frgr_mem */ - parse_frgr_mem(us, - p[1] >> 4, - ((p[1] & 0xf) << 16) - | (p[2] << 8) | p[3]); - p += 3; - } else if ((b >> 5) == 6) { - /* P6 */ - if (b & 0x10) - parse_gr_mem(us, b & 0xf); - else - parse_fr_mem(us, b & 0xf); - p++; - } else if ((b >> 4) == 14) { - /* P7 */ - int x, y; - p++; - x = read_uleb128(&p); - switch (b & 0xf) { - case 0: - y = read_uleb128(&p); - parse_mem_stack_f(us, x, y); - break; - case 1: - parse_mem_stack_v(us, x); - break; - case 2: - parse_spill_base(us, x); - break; - case 3: - parse_psp_sprel(us, x); - break; - case 4: - parse_rp_when(us, x); - break; - case 5: - parse_rp_psprel(us, x); - break; - case 6: - parse_pfs_when(us, x); - break; - case 7: - parse_pfs_psprel(us, x); - break; - case 8: - parse_preds_when(us, x); - break; - case 9: - parse_preds_psprel(us, x); - break; - case 10: - parse_lc_when(us, x); - break; - case 11: - parse_lc_psprel(us, x); - break; - case 12: - parse_unat_when(us, x); - break; - case 13: - parse_unat_psprel(us, x); - break; - case 14: - parse_fpsr_when(us, x); - break; - case 15: - parse_fpsr_psprel(us, x); - break; - } - } else if ((b >> 0) == 240) { - /* P8 */ - int which = p[1]; - int x; - p += 2; - x = read_uleb128(&p); - switch (which) { - case 1: - parse_rp_sprel(us, x); - break; - case 2: - parse_pfs_sprel(us, x); - break; - case 3: - parse_preds_sprel(us, x); - break; - case 4: - parse_lc_sprel(us, x); - break; - case 5: - parse_unat_sprel(us, x); - break; - case 6: - parse_fpsr_sprel(us, x); - break; - case 7: - parse_bsp_when(us, x); - break; - case 8: - parse_bsp_psprel(us, x); - break; - case 9: - parse_bsp_sprel(us, x); - break; - case 10: - parse_bspstore_when(us, x); - break; - case 11: - parse_bspstore_psprel(us, x); - break; - case 12: - parse_bspstore_sprel(us, x); - break; - case 13: - parse_rnat_when(us, x); - break; - case 14: - parse_rnat_psprel(us, x); - break; - case 15: - parse_rnat_sprel(us, x); - break; - case 16: - parse_priunat_when_gr(us, x); - break; - case 17: - parse_priunat_psprel(us, x); - break; - case 18: - parse_priunat_sprel(us, x); - break; - case 19: - parse_priunat_when_mem(us, x); - break; - } - } else if ((b >> 0) == 241) { - /* P9 */ - parse_gr_gr(us, p[1], p[2]); - p += 3; - } else if ((b >> 0) == 255) { - /* P10 (ignored) */ - p += 3; - } - } else { - if ((b >> 6) == 2) { - /* B1 */ - if ((b & (1 << 5)) == 0) - parse_label_state(us, - b & 0x1f); - else - parse_copy_state(us, - b & 0x1f); - } else if ((b >> 5) == 6) { - /* B2 */ - int ecount = b & 0x1f; - int t; - p++; - t = read_uleb128(&p); - parse_epilogue(us, t, ecount); - } else if ((b >> 0) == 224) { - /* B3 */ - int t, ecount; - p++; - t = read_uleb128(&p); - ecount = read_uleb128(&p); - parse_epilogue(us, t, ecount); - } else if ((b >> 4) == 15) { - /* B4 */ - int label; - p++; - label = read_uleb128(&p); - if ((b & (1 << 3)) == 0) - parse_label_state(us, label); - else - parse_copy_state(us, label); - } - } - /* - * X records can appear in both prologue and - * body. - */ - if ((b >> 0) == 249) { - /* X1 */ - int r, reg, t, off; - r = (p[1] >> 7) & 1; - reg = p[1] & 0x7f; - p += 2; - t = read_uleb128(&p); - off = read_uleb128(&p); - if (r == 0) - parse_spill_psprel(us, t, reg, off); - else - parse_spill_sprel(us, t, reg, off); - } else if ((b >> 0) == 250) { - /* X2 */ - int reg, treg, t; - reg = p[1] & 0x7f; - treg = p[2] | ((p[1] & 0x80) << 1); - p += 3; - t = read_uleb128(&p); - parse_spill_reg(us, t, reg, treg); - } else if ((b >> 0) == 251) { - /* X3 */ - int r, qp, reg, t, off; - r = (p[1] >> 7) & 1; - qp = p[1] & 0x3f; - reg = p[2]; - p += 3; - t = read_uleb128(&p); - off = read_uleb128(&p); - if (r == 0) - parse_spill_psprel_p(us, t, qp, - reg, off); - else - parse_spill_sprel_p(us, t, qp, - reg, off); - } else if ((b >> 0) == 252) { - /* X4 */ - int qp, reg, treg, t; - qp = p[1] & 0x3f; - reg = p[2] & 0x7f; - treg = p[3] | ((p[2] & 0x80) << 1); - p += 4; - t = read_uleb128(&p); - parse_spill_reg_p(us, t, qp, reg, treg); - } - } - } - - noentry: - /* - * Now that we have worked out suitable values for rp, ar.pfs - * and sp, we can shift state to the previous function. If we - * haven't managed to figure out a new value for ip, then we - * assume that the unwinding didn't work and return an error. - */ - if (us->us_ip == us->us_regs.rs_br[0].ur_value) - return EINVAL; - us->us_ip = us->us_regs.rs_br[0].ur_value; - us->us_ri = 0; - us->us_cfm = us->us_regs.rs_pfs.ur_value; - DPF(("new cfm is 0x%lx\n", us->us_cfm)); - us->us_bsp = ia64_rse_previous_frame(us->us_bsp, - (us->us_cfm >> 7) & 0x7f); - DPF(("new bsp is %p\n", us->us_bsp)); - us->us_regs.rs_gr[12] = us->us_regs.rs_psp; - return 0; + LIST_INIT(&unw_tables); + uwx_register_alloc_cb(unw_alloc, unw_free); } +SYSINIT(unwind, SI_SUB_KMEM, SI_ORDER_ANY, unw_initialize, 0); diff --git a/sys/ia64/ia64/vm_machdep.c b/sys/ia64/ia64/vm_machdep.c index 5c5fbff..d0ecb6e 100644 --- a/sys/ia64/ia64/vm_machdep.c +++ b/sys/ia64/ia64/vm_machdep.c @@ -127,170 +127,67 @@ cpu_set_upcall_kse(struct thread *td, struct kse_upcall *ku) * ready to run and return to user mode. */ void -cpu_fork(td1, p2, td2, flags) - register struct thread *td1; - register struct proc *p2; - register struct thread *td2; - int flags; +cpu_fork(struct thread *td1, struct proc *p2 __unused, struct thread *td2, + int flags) { - struct proc *p1; - struct trapframe *p2tf; - u_int64_t bspstore, *p1bs, *p2bs, rnatloc, rnat; + char *stackp; KASSERT(td1 == curthread || td1 == &thread0, - ("cpu_fork: p1 not curproc and not proc0")); + ("cpu_fork: td1 not curthread and not thread0")); if ((flags & RFPROC) == 0) return; - p1 = td1->td_proc; - td2->td_pcb = (struct pcb *) - (td2->td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; - td2->td_md.md_flags = td1->td_md.md_flags & (MDP_FPUSED | MDP_UAC_MASK); - - /* - * Copy floating point state from the FP chip to the PCB - * if this process has state stored there. - */ - ia64_fpstate_save(td1, 0); - - /* - * Copy pcb and stack from proc p1 to p2. We do this as - * cheaply as possible, copying only the active part of the - * stack. The stack and pcb need to agree. Make sure that the - * new process has FEN disabled. - */ - bcopy(td1->td_pcb, td2->td_pcb, sizeof(struct pcb)); - /* - * Set the floating point state. + * Save the preserved registers and the high FP registers in the + * PCB if we're the parent (ie td1 == curthread) so that we have + * a valid PCB. This also causes a RSE flush. We don't have to + * do that otherwise, because there wouldn't be anything important + * to save. */ -#if 0 - if ((td2->td_pcb->pcb_fp_control & IEEE_INHERIT) == 0) { - td2->td_pcb->pcb_fp_control = 0; - td2->td_pcb->pcb_fp.fpr_cr = (FPCR_DYN_NORMAL - | FPCR_INVD | FPCR_DZED - | FPCR_OVFD | FPCR_INED - | FPCR_UNFD); + if (td1 == curthread) { + if (savectx(td1->td_pcb) != 0) + panic("unexpected return from savectx()"); + ia64_highfp_save(td1); } -#endif /* - * Arrange for a non-local goto when the new process - * is started, to resume here, returning nonzero from setjmp. + * create the child's kernel stack and backing store. We basicly + * create an image of the parent's stack and backing store and + * adjust where necessary. */ -#ifdef DIAGNOSTIC - if (td1 == curthread) - ia64_fpstate_check(td1); -#endif + stackp = (char *)(td2->td_kstack + KSTACK_PAGES * PAGE_SIZE); - /* - * create the child's kernel stack, from scratch. - * - * Pick a stack pointer, leaving room for a trapframe; - * copy trapframe from parent so return to user mode - * will be to right address, with correct registers. Clear the - * high-fp enable for the new process so that it is forced to - * load its state from the pcb. - */ - td2->td_frame = (struct trapframe *)td2->td_pcb - 1; - bcopy(td1->td_frame, td2->td_frame, sizeof(struct trapframe)); - td2->td_frame->tf_cr_ipsr |= IA64_PSR_DFH; - - /* - * Set up return-value registers as fork() libc stub expects. - */ - p2tf = td2->td_frame; - if (p2tf->tf_cr_ipsr & IA64_PSR_IS) { - p2tf->tf_r[FRAME_R8] = 0; /* child returns zero (eax) */ - p2tf->tf_r[FRAME_R10] = 1; /* is child (edx) */ - td2->td_pcb->pcb_ar_eflag &= ~PSL_C; /* no error */ - } else { - p2tf->tf_r[FRAME_R8] = 0; /* child's pid (linux) */ - p2tf->tf_r[FRAME_R9] = 1; /* is child (FreeBSD) */ - p2tf->tf_r[FRAME_R10] = 0; /* no error */ - } + stackp -= sizeof(struct pcb); + td2->td_pcb = (struct pcb *)stackp; + bcopy(td1->td_pcb, td2->td_pcb, sizeof(struct pcb)); - /* - * Turn off RSE for a moment and work out our current - * ar.bspstore. This assumes that td1==curthread. Also - * flush dirty regs to ensure that the user's stacked - * regs are written out to backing store. - * - * We could cope with td1!=curthread by digging values - * out of its PCB but I don't see the point since - * current usage only allows &thread0 when creating kernel - * threads and &thread0 doesn't have any dirty regs. - */ + stackp -= sizeof(struct trapframe); + td2->td_frame = (struct trapframe *)stackp; + bcopy(td1->td_frame, td2->td_frame, sizeof(struct trapframe)); + td2->td_frame->tf_length = sizeof(struct trapframe); - p1bs = (u_int64_t *)td1->td_kstack; - p2bs = (u_int64_t *)td2->td_kstack; + bcopy((void*)td1->td_kstack, (void*)td2->td_kstack, + td2->td_frame->tf_special.ndirty); - if (td1 == curthread) { - __asm __volatile("mov ar.rsc=0;;"); - __asm __volatile("flushrs;;" ::: "memory"); - __asm __volatile("mov %0=ar.bspstore" : "=r"(bspstore)); + /* Set-up the return values as expected by the fork() libc stub. */ + if (td2->td_frame->tf_special.psr & IA64_PSR_IS) { + td2->td_frame->tf_scratch.gr8 = 0; + td2->td_frame->tf_scratch.gr10 = 1; } else { - bspstore = (u_int64_t) p1bs; + td2->td_frame->tf_scratch.gr8 = 0; + td2->td_frame->tf_scratch.gr9 = 1; + td2->td_frame->tf_scratch.gr10 = 0; } - /* - * Copy enough of td1's backing store to include all - * the user's stacked regs. - */ - bcopy(p1bs, p2bs, td1->td_frame->tf_ndirty); - /* - * To calculate the ar.rnat for td2, we need to decide - * if td1's ar.bspstore has advanced past the place - * where the last ar.rnat which covers the user's - * saved registers would be placed. If so, we read - * that one from memory, otherwise we take td1's - * current ar.rnat. If we are simply spawning a new kthread - * from &thread0 we don't care about ar.rnat. - */ - if (td1 == curthread) { - rnatloc = (u_int64_t)p1bs + td1->td_frame->tf_ndirty; - rnatloc |= 0x1f8; - if (bspstore > rnatloc) - rnat = *(u_int64_t *) rnatloc; - else - __asm __volatile("mov %0=ar.rnat;;" : "=r"(rnat)); - - /* - * Switch the RSE back on. - */ - __asm __volatile("mov ar.rsc=3;;"); - } else { - rnat = 0; - } - - /* - * Setup the child's pcb so that its ar.bspstore - * starts just above the region which we copied. This - * should work since the child will normally return - * straight into exception_restore. Also initialise its - * pmap to the containing proc's vmspace. - */ - td2->td_pcb->pcb_ar_bsp = (u_int64_t)p2bs + td1->td_frame->tf_ndirty; - td2->td_pcb->pcb_ar_rnat = rnat; - td2->td_pcb->pcb_ar_pfs = 0; - td2->td_pcb->pcb_current_pmap = (u_int64_t) - vmspace_pmap(td2->td_proc->p_vmspace); + td2->td_pcb->pcb_special.bspstore = td2->td_kstack + + td2->td_frame->tf_special.ndirty; + td2->td_pcb->pcb_special.pfs = 0; + td2->td_pcb->pcb_current_pmap = vmspace_pmap(td2->td_proc->p_vmspace); - /* - * Arrange for continuation at fork_return(), which - * will return to exception_restore(). Note that the - * child process doesn't stay in the kernel for long! - * - * The extra 16 bytes subtracted from sp is part of the ia64 - * ABI - a function can assume that the 16 bytes above sp are - * available as scratch space. - */ - td2->td_pcb->pcb_sp = (u_int64_t)p2tf - 16; - td2->td_pcb->pcb_r[PCB_R4] = (u_int64_t)fork_return; - td2->td_pcb->pcb_r[PCB_R5] = FDESC_FUNC(exception_restore); - td2->td_pcb->pcb_r[PCB_R6] = (u_int64_t)td2; - td2->td_pcb->pcb_rp = FDESC_FUNC(fork_trampoline); + td2->td_pcb->pcb_special.sp = (uintptr_t)stackp - 16; + td2->td_pcb->pcb_special.rp = FDESC_FUNC(fork_trampoline); + cpu_set_fork_handler(td2, (void (*)(void*))fork_return, td2); } /* @@ -305,8 +202,8 @@ cpu_set_fork_handler(td, func, arg) void (*func)(void *); void *arg; { - td->td_pcb->pcb_r[PCB_R4] = (u_int64_t) func; - td->td_pcb->pcb_r[PCB_R6] = (u_int64_t) arg; + td->td_frame->tf_scratch.gr2 = (u_int64_t)func; + td->td_frame->tf_scratch.gr3 = (u_int64_t)arg; } /* @@ -315,11 +212,11 @@ cpu_set_fork_handler(td, func, arg) * When the proc is reaped, cpu_wait() will gc the VM state. */ void -cpu_exit(td) - register struct thread *td; +cpu_exit(struct thread *td) { - ia64_fpstate_drop(td); + /* Throw away the high FP registers. */ + ia64_highfp_drop(td); } void diff --git a/sys/ia64/include/asm.h b/sys/ia64/include/asm.h index f0a1ec2..8be7b8a 100644 --- a/sys/ia64/include/asm.h +++ b/sys/ia64/include/asm.h @@ -151,15 +151,22 @@ _name_ = _value_ label: ASCIZ msg; \ .text; + /* * System call glue. */ -#define SYSCALLNUM(name) \ - SYS_ ## name +#define SYSCALLNUM(name) SYS_ ## name + +#define CALLSYS_NOERROR(name) \ +{ .mmi ; \ + alloc r9 = ar.pfs, 0, 0, 8, 0 ; \ + mov r31 = ar.k5 ; \ + mov r10 = b0 ;; } \ +{ .mib ; \ + mov r8 = SYSCALLNUM(name) ; \ + mov b7 = r31 ; \ + br.call.sptk b0 = b7 ;; } -#define CALLSYS_NOERROR(name) \ - mov r15=SYSCALLNUM(name); \ - break 0x100000 ;; /* * WEAK_ALIAS: create a weak alias (ELF only). diff --git a/sys/ia64/include/cpu.h b/sys/ia64/include/cpu.h index 324b035..c2bd78c 100644 --- a/sys/ia64/include/cpu.h +++ b/sys/ia64/include/cpu.h @@ -46,28 +46,25 @@ #ifndef _MACHINE_CPU_H_ #define _MACHINE_CPU_H_ -/* - * Exported definitions unique to Alpha cpu support. - */ - #include <machine/frame.h> -#define cpu_getstack(td) ((td)->td_frame->tf_r[FRAME_SP]) - /* - * Arguments to hardclock and gatherstats encapsulate the previous - * machine state in an opaque clockframe. One the Alpha, we use - * what we push on an interrupt (a trapframe). + * Arguments to hardclock and gatherstats encapsulate the previous machine + * state in an opaque clockframe. */ struct clockframe { - struct trapframe cf_tf; + struct trapframe cf_tf; }; -#define TRAPF_USERMODE(framep) \ - (((framep)->tf_cr_ipsr & IA64_PSR_CPL) == IA64_PSR_CPL_USER) -#define TRAPF_PC(framep) ((framep)->tf_cr_iip) +#define CLKF_PC(cf) ((cf)->cf_tf.tf_special.iip) +#define CLKF_USERMODE(cf) ((CLKF_PC(cf) >> 61) < 5) -#define CLKF_USERMODE(framep) TRAPF_USERMODE(&(framep)->cf_tf) -#define CLKF_PC(framep) TRAPF_PC(&(framep)->cf_tf) +/* Used by signaling code. */ +#define cpu_getstack(td) ((td)->td_frame->tf_special.sp) + +/* XXX */ +#define TRAPF_PC(tf) ((tf)->tf_special.iip) +#define TRAPF_USERMODE(framep) \ + (((framep)->tf_special.psr & IA64_PSR_CPL) == IA64_PSR_CPL_USER) /* * CTL_MACHDEP definitions. @@ -107,34 +104,33 @@ struct trapframe; extern struct rpb *hwrpb; extern volatile int mc_expected, mc_received; -int badaddr (void *, size_t); +int badaddr(void *, size_t); int badaddr_read(void *, size_t, void *); u_int64_t console_restart(u_int64_t, u_int64_t, u_int64_t); -void do_sir(void); +int do_ast(struct trapframe *); void dumpconf(void); -void exception_restore(void); /* MAGIC */ void frametoreg(struct trapframe *, struct reg *); long fswintrberr(void); /* MAGIC */ -int ia64_pa_access(u_long); +int ia64_highfp_drop(struct thread *); +int ia64_highfp_load(struct thread *); +int ia64_highfp_save(struct thread *); void ia64_init(u_int64_t, u_int64_t); -void ia64_fpstate_check(struct thread *p); -void ia64_fpstate_save(struct thread *p, int write); -void ia64_fpstate_drop(struct thread *p); -void ia64_fpstate_switch(struct thread *p); +int ia64_pa_access(u_long); void init_prom_interface(struct rpb*); void interrupt(u_int64_t, struct trapframe *); -void machine_check - (unsigned long, struct trapframe *, unsigned long, unsigned long); +void machine_check(unsigned long, struct trapframe *, unsigned long, + unsigned long); u_int64_t hwrpb_checksum(void); void hwrpb_restart_setup(void); void regdump(struct trapframe *); void regtoframe(struct reg *, struct trapframe *); void set_iointr(void (*)(void *, unsigned long)); void fork_trampoline(void); /* MAGIC */ -void syscall(int, u_int64_t *, struct trapframe *); -void trap(int vector, int imm, struct trapframe *framep); +int syscall(struct trapframe *); +void trap(int vector, struct trapframe *framep); void ia64_probe_sapics(void); int ia64_count_cpus(void); +void map_gateway_page(void); void map_pal_code(void); void map_port_space(void); void cpu_mp_add(uint, uint, uint); diff --git a/sys/ia64/include/db_machdep.h b/sys/ia64/include/db_machdep.h index 9a54356..4b801f6 100644 --- a/sys/ia64/include/db_machdep.h +++ b/sys/ia64/include/db_machdep.h @@ -50,8 +50,8 @@ typedef struct trapframe db_regs_t; extern db_regs_t ddb_regs; /* register state */ #define DDB_REGS (&ddb_regs) -#define PC_REGS(regs) ((db_addr_t)(regs)->tf_cr_iip \ - + (((regs)->tf_cr_ipsr >> 41) & 3)) +#define PC_REGS(regs) ((db_addr_t)(regs)->tf_special.iip + \ + (((regs)->tf_special.psr >> 41) & 3)) #define BKPT_WRITE(addr, storage) db_write_breakpoint(addr, storage) #define BKPT_CLEAR(addr, storage) db_clear_breakpoint(addr, storage) @@ -59,8 +59,8 @@ extern db_regs_t ddb_regs; /* register state */ #define BKPT_SKIP db_skip_breakpoint() -#define db_clear_single_step(regs) ddb_regs.tf_cr_ipsr &= ~IA64_PSR_SS -#define db_set_single_step(regs) ddb_regs.tf_cr_ipsr |= IA64_PSR_SS +#define db_clear_single_step(regs) ddb_regs.tf_special.psr &= ~IA64_PSR_SS +#define db_set_single_step(regs) ddb_regs.tf_special.psr |= IA64_PSR_SS #define IS_BREAKPOINT_TRAP(type, code) (type == IA64_VEC_BREAK) #define IS_WATCHPOINT_TRAP(type, code) 0 diff --git a/sys/ia64/include/frame.h b/sys/ia64/include/frame.h index a66345d..262a8bf 100644 --- a/sys/ia64/include/frame.h +++ b/sys/ia64/include/frame.h @@ -29,87 +29,18 @@ #ifndef _MACHINE_FRAME_H_ #define _MACHINE_FRAME_H_ -#include <machine/reg.h> +#include <machine/_regset.h> /* * Software trap, exception, and syscall frame. */ struct trapframe { - u_int64_t tf_flags; + uint64_t tf_length; + uint64_t tf_flags; #define FRAME_SYSCALL 1 /* syscalls use a partial trapframe */ - - u_int64_t tf_cr_iip; - u_int64_t tf_cr_ipsr; - u_int64_t tf_cr_isr; - u_int64_t tf_cr_ifa; - u_int64_t tf_pr; - u_int64_t tf_ar_rsc; - u_int64_t tf_ar_pfs; - u_int64_t tf_cr_ifs; - u_int64_t tf_ar_bspstore; - u_int64_t tf_ar_rnat; - u_int64_t tf_ndirty; - u_int64_t tf_ar_unat; - u_int64_t tf_ar_ccv; - u_int64_t tf_ar_fpsr; - u_int64_t tf_ar_lc; - u_int64_t tf_ar_ec; - - u_int64_t tf_b[8]; - - u_int64_t tf_r[31]; /* don't need to save r0 */ -#define FRAME_R1 0 -#define FRAME_R2 1 -#define FRAME_R3 2 -#define FRAME_R4 3 -#define FRAME_R5 4 -#define FRAME_R6 5 -#define FRAME_R7 6 -#define FRAME_R8 7 -#define FRAME_R9 8 -#define FRAME_R10 9 -#define FRAME_R11 10 -#define FRAME_R12 11 -#define FRAME_R13 12 -#define FRAME_R14 13 -#define FRAME_R15 14 -#define FRAME_R16 15 -#define FRAME_R17 16 -#define FRAME_R18 17 -#define FRAME_R19 18 -#define FRAME_R20 19 -#define FRAME_R21 20 -#define FRAME_R22 21 -#define FRAME_R23 22 -#define FRAME_R24 23 -#define FRAME_R25 24 -#define FRAME_R26 25 -#define FRAME_R27 26 -#define FRAME_R28 27 -#define FRAME_R29 28 -#define FRAME_R30 29 -#define FRAME_R31 30 - -#define FRAME_GP FRAME_R1 -#define FRAME_SP FRAME_R12 -#define FRAME_TP FRAME_R13 - - /* - * We rely on the compiler to save/restore f2-f5 and - * f16-f31. We also tell the compiler to avoid f32-f127 - * completely so we don't worry about them at all. - */ - struct ia64_fpreg tf_f[10]; -#define FRAME_F6 0 -#define FRAME_F7 1 -#define FRAME_F8 2 -#define FRAME_F9 3 -#define FRAME_F10 4 -#define FRAME_F11 5 -#define FRAME_F12 6 -#define FRAME_F13 7 -#define FRAME_F14 8 -#define FRAME_F15 9 + struct _special tf_special; + struct _caller_saved tf_scratch; + struct _caller_saved_fp tf_scratch_fp; }; #endif /* _MACHINE_FRAME_H_ */ diff --git a/sys/ia64/include/pcb.h b/sys/ia64/include/pcb.h index c5ee145..ac69409 100644 --- a/sys/ia64/include/pcb.h +++ b/sys/ia64/include/pcb.h @@ -1,4 +1,5 @@ /*- + * Copyright (c) 2003 Doug Rabson * Copyright (c) 2000 Doug Rabson * All rights reserved. * @@ -29,72 +30,44 @@ #ifndef _MACHINE_PCB_H_ #define _MACHINE_PCB_H_ +#include <machine/_regset.h> + /* * PCB: process control block */ +struct pmap; struct pcb { - uint64_t pcb_sp; - uint64_t pcb_ar_unat; - uint64_t pcb_rp; - uint64_t pcb_pr; - struct ia64_fpreg pcb_f[20]; -#define PCB_F2 0 -#define PCB_F3 1 -#define PCB_F4 2 -#define PCB_F5 3 -#define PCB_F16 4 -#define PCB_F17 5 -#define PCB_F18 6 -#define PCB_F19 7 -#define PCB_F20 8 -#define PCB_F21 9 -#define PCB_F22 10 -#define PCB_F23 11 -#define PCB_F24 12 -#define PCB_F25 13 -#define PCB_F26 14 -#define PCB_F27 15 -#define PCB_F28 16 -#define PCB_F29 17 -#define PCB_F30 18 -#define PCB_F31 19 - uint64_t pcb_r[4]; -#define PCB_R4 0 -#define PCB_R5 1 -#define PCB_R6 2 -#define PCB_R7 3 - uint64_t pcb_unat47; - uint64_t pcb_b[5]; -#define PCB_B1 0 -#define PCB_B2 1 -#define PCB_B3 2 -#define PCB_B4 3 -#define PCB_B5 4 - uint64_t pcb_ar_bsp; - uint64_t pcb_ar_pfs; - uint64_t pcb_ar_rnat; - uint64_t pcb_ar_lc; - - uint64_t pcb_current_pmap; - - uint64_t pcb_ar_fcr; - uint64_t pcb_ar_eflag; - uint64_t pcb_ar_csd; - uint64_t pcb_ar_ssd; - uint64_t pcb_ar_fsr; - uint64_t pcb_ar_fir; - uint64_t pcb_ar_fdr; - - /* Aligned! */ - struct ia64_fpreg pcb_highfp[96]; /* f32-f127 */ + struct _special pcb_special; + struct _callee_saved pcb_preserved; + struct _callee_saved_fp pcb_preserved_fp; + struct _high_fp pcb_high_fp; + struct pcpu *pcb_fpcpu; + struct pmap *pcb_current_pmap; uint64_t pcb_onfault; /* for copy faults */ uint64_t pcb_accessaddr; /* for [fs]uswintr */ + +#if IA32 + uint64_t pcb_ia32_cflg; + uint64_t pcb_ia32_eflag; + uint64_t pcb_ia32_fcr; + uint64_t pcb_ia32_fdr; + uint64_t pcb_ia32_fir; + uint64_t pcb_ia32_fsr; +#endif }; #ifdef _KERNEL -void restorectx(struct pcb *); -void savectx(struct pcb *); + +#define savectx(p) swapctx(p, NULL) +void restorectx(struct pcb *) __dead2; +int swapctx(struct pcb *old, struct pcb *new); + +#if IA32 +void ia32_restorectx(struct pcb *); +void ia32_savectx(struct pcb *); +#endif + #endif #endif /* _MACHINE_PCB_H_ */ diff --git a/sys/ia64/include/proc.h b/sys/ia64/include/proc.h index 4daead6..01ba3e8 100644 --- a/sys/ia64/include/proc.h +++ b/sys/ia64/include/proc.h @@ -37,7 +37,6 @@ struct mdthread { u_long md_flags; - void *md_kstackvirt; /* virtual address of td_kstack */ vm_offset_t md_bspstore; /* initial ar.bspstore */ register_t md_savecrit; }; @@ -50,7 +49,7 @@ struct mdthread { #define MDP_UAC_MASK (MDP_UAC_NOPRINT | MDP_UAC_NOFIX | MDP_UAC_SIGBUS) struct mdproc { - struct user *md_uservirt; /* virtual address of p_addr */ + int __dummy; /* Avoid having an empty struct. */ }; #endif /* !_MACHINE_PROC_H_ */ diff --git a/sys/ia64/include/reg.h b/sys/ia64/include/reg.h index 8ba6e7d..d453826 100644 --- a/sys/ia64/include/reg.h +++ b/sys/ia64/include/reg.h @@ -29,37 +29,18 @@ #ifndef _MACHINE_REG_H_ #define _MACHINE_REG_H_ -#ifndef _IA64_FPREG_DEFINED - -struct ia64_fpreg { - uint64_t fpr_bits[2]; -} __aligned(16); - -#define _IA64_FPREG_DEFINED - -#endif +#include <machine/_regset.h> struct reg { - uint64_t r_gr[128]; - uint64_t r_br[8]; - uint64_t r_cfm; - uint64_t r_ip; /* Bits 0-3 encode the slot number */ - uint64_t r_pr; - uint64_t r_psr; /* User mask */ - uint64_t r_ar_rsc; - uint64_t r_ar_bsp; - uint64_t r_ar_bspstore; - uint64_t r_ar_rnat; - uint64_t r_ar_ccv; - uint64_t r_ar_unat; - uint64_t r_ar_fpsr; - uint64_t r_ar_pfs; - uint64_t r_ar_lc; - uint64_t r_ar_ec; + struct _special r_special; + struct _callee_saved r_preserved; + struct _caller_saved r_scratch; }; struct fpreg { - struct ia64_fpreg fpr_regs[128]; + struct _callee_saved_fp fpr_preserved; + struct _caller_saved_fp fpr_scratch; + struct _high_fp fpr_high; }; struct dbreg { @@ -68,15 +49,9 @@ struct dbreg { }; #ifdef _KERNEL - struct thread; -void restorehighfp(struct ia64_fpreg *); -void savehighfp(struct ia64_fpreg *); - -/* - * XXX these interfaces are MI, so they should be declared in a MI place. - */ +/* XXX these interfaces are MI, so they should be declared in a MI place. */ int fill_regs(struct thread *, struct reg *); int set_regs(struct thread *, struct reg *); int fill_fpregs(struct thread *, struct fpreg *); diff --git a/sys/ia64/include/signal.h b/sys/ia64/include/signal.h index 1a20e01..025b979 100644 --- a/sys/ia64/include/signal.h +++ b/sys/ia64/include/signal.h @@ -51,62 +51,35 @@ typedef long sig_atomic_t; #endif #if __XSI_VISIBLE -/* - * Minimum signal stack size. The current signal frame - * for IA-64 is 2656 bytes large. - */ +/* Minimum signal stack size. */ #define MINSIGSTKSZ (3072 * 4) #endif -#if __BSD_VISIBLE -#ifndef _IA64_FPREG_DEFINED - -struct ia64_fpreg { - unsigned long fpr_bits[2]; -} __aligned(16); - -#define _IA64_FPREG_DEFINED - -#endif -#endif - - /* * Information pushed on stack when a signal is delivered. * This is used by the kernel to restore state following * execution of the signal handler. It is also made available * to the handler to allow it to restore state properly if * a non-standard exit is performed. - * - * Note that sc_regs[] and sc_fpregs[]+sc_fpcr are inline - * representations of 'struct reg' and 'struct fpreg', respectively. */ #if __BSD_VISIBLE +#include <machine/_regset.h> + /* * The sequence of the fields should match those in * mcontext_t. Keep them in sync! */ struct sigcontext { - struct __sigset sc_mask; /* signal mask to restore */ - unsigned long sc_onstack; - unsigned long sc_flags; - unsigned long sc_nat; - unsigned long sc_sp; - unsigned long sc_ip; - unsigned long sc_cfm; - unsigned long sc_um; - unsigned long sc_ar_rsc; - unsigned long sc_ar_bsp; - unsigned long sc_ar_rnat; - unsigned long sc_ar_ccv; - unsigned long sc_ar_unat; - unsigned long sc_ar_fpsr; - unsigned long sc_ar_pfs; - unsigned long sc_pr; - unsigned long sc_br[8]; - unsigned long sc_gr[32]; - struct ia64_fpreg sc_fr[128]; + struct __sigset sc_mask; /* signal mask to restore */ + unsigned long sc_onstack; + unsigned long sc_flags; + struct _special sc_special; + struct _callee_saved sc_preserved; + struct _callee_saved_fp sc_preserved_fp; + struct _caller_saved sc_scratch; + struct _caller_saved_fp sc_scratch_fp; + struct _high_fp sc_high_fp; }; #endif /* __BSD_VISIBLE */ diff --git a/sys/ia64/include/smp.h b/sys/ia64/include/smp.h index e985fc0..daf5911 100644 --- a/sys/ia64/include/smp.h +++ b/sys/ia64/include/smp.h @@ -14,15 +14,16 @@ */ /* Architecture specific IPIs. */ #define IPI_AP_WAKEUP 0 -#define IPI_MCA_RENDEZ 1 +#define IPI_HIGH_FP 1 #define IPI_MCA_CMCV 2 -#define IPI_TEST 3 +#define IPI_MCA_RENDEZ 3 +#define IPI_TEST 4 /* Machine independent IPIs. */ -#define IPI_AST 4 -#define IPI_RENDEZVOUS 5 -#define IPI_STOP 6 +#define IPI_AST 5 +#define IPI_RENDEZVOUS 6 +#define IPI_STOP 7 -#define IPI_COUNT 7 +#define IPI_COUNT 8 #ifndef LOCORE @@ -32,6 +33,7 @@ void ipi_all(int ipi); void ipi_all_but_self(int ipi); void ipi_selected(u_int64_t cpus, int ipi); void ipi_self(int ipi); +void ipi_send(u_int64_t lid, int ipi); #endif /* !LOCORE */ #endif /* _KERNEL */ diff --git a/sys/ia64/include/ucontext.h b/sys/ia64/include/ucontext.h index e8660a5..fc32b76 100644 --- a/sys/ia64/include/ucontext.h +++ b/sys/ia64/include/ucontext.h @@ -31,35 +31,19 @@ #ifndef _MACHINE_UCONTEXT_H_ #define _MACHINE_UCONTEXT_H_ -#define IA64_MC_FLAG_ONSTACK 0 -#define IA64_MC_FLAG_IN_SYSCALL 1 -#define IA64_MC_FLAG_FPH_VALID 2 +#include <machine/_regset.h> typedef struct __mcontext { - /* - * These fields must match the definition - * of struct sigcontext. That way we can support - * struct sigcontext and ucontext_t at the same - * time. - */ - long mc_onstack; /* XXX - sigcontext compat. */ - unsigned long mc_flags; - unsigned long mc_nat; - unsigned long mc_sp; - unsigned long mc_ip; - unsigned long mc_cfm; - unsigned long mc_um; - unsigned long mc_ar_rsc; - unsigned long mc_ar_bsp; - unsigned long mc_ar_rnat; - unsigned long mc_ar_ccv; - unsigned long mc_ar_unat; - unsigned long mc_ar_fpsr; - unsigned long mc_ar_pfs; - unsigned long mc_pr; - unsigned long mc_br[8]; - unsigned long mc_gr[32]; - struct ia64_fpreg mc_fr[128]; + uint64_t mc_flags; +#define IA64_MC_FLAGS_SCRATCH_VALID 1 +#define IA64_MC_FLAGS_HIGHFP_VALID 2 + uint64_t _reserved_; + struct _special mc_special; + struct _callee_saved mc_preserved; + struct _callee_saved_fp mc_preserved_fp; + struct _caller_saved mc_scratch; + struct _caller_saved_fp mc_scratch_fp; + struct _high_fp mc_high_fp; } mcontext_t; #endif /* !_MACHINE_UCONTEXT_H_ */ diff --git a/sys/ia64/include/unwind.h b/sys/ia64/include/unwind.h index 3509c05..1cf9f4a 100644 --- a/sys/ia64/include/unwind.h +++ b/sys/ia64/include/unwind.h @@ -26,13 +26,25 @@ * $FreeBSD$ */ -int ia64_add_unwind_table(vm_offset_t, vm_offset_t, vm_offset_t); -void ia64_delete_unwind_table(vm_offset_t); +#ifndef _MACHINE_UNWIND_H_ +#define _MACHINE_UNWIND_H_ -struct ia64_unwind_state *ia64_create_unwind_state(struct trapframe *framep); -void ia64_free_unwind_state(struct ia64_unwind_state *us); -u_int64_t ia64_unwind_state_get_ip(struct ia64_unwind_state *us); -u_int64_t ia64_unwind_state_get_sp(struct ia64_unwind_state *us); -u_int64_t ia64_unwind_state_get_cfm(struct ia64_unwind_state *us); -u_int64_t *ia64_unwind_state_get_bsp(struct ia64_unwind_state *us); -int ia64_unwind_state_previous_frame(struct ia64_unwind_state *us); +struct uwx_env; + +struct unw_regstate { + struct trapframe *frame; + struct uwx_env *env; + uint64_t keyval[8]; +}; + +int unw_create(struct unw_regstate *s, struct trapframe *tf); +int unw_step(struct unw_regstate *s); + +int unw_get_bsp(struct unw_regstate *s, uint64_t *r); +int unw_get_cfm(struct unw_regstate *s, uint64_t *r); +int unw_get_ip(struct unw_regstate *s, uint64_t *r); + +int unw_table_add(uint64_t, uint64_t, uint64_t); +void unw_table_remove(uint64_t); + +#endif /* _MACHINE_UNWIND_H_ */ diff --git a/sys/ia64/include/vmparam.h b/sys/ia64/include/vmparam.h index b9004db..69480a3 100644 --- a/sys/ia64/include/vmparam.h +++ b/sys/ia64/include/vmparam.h @@ -50,11 +50,10 @@ /* * USRTEXT is the start of the user text/data space, while USRSTACK * is the top (end) of the user stack. Immediately above the user stack - * resides the user structure, which is UPAGES long and contains the - * kernel stack. + * resides the syscall gateway page. */ #define USRTEXT CLBYTES -#define USRSTACK VM_MAXUSER_ADDRESS +#define USRSTACK VM_MAX_ADDRESS /* * Virtual memory related constants, all in bytes @@ -140,12 +139,13 @@ /* user/kernel map constants */ #define VM_MIN_ADDRESS 0 -#define VM_MAXUSER_ADDRESS IA64_RR_BASE(5) -#define VM_MAX_ADDRESS VM_MAXUSER_ADDRESS -#define VM_MIN_KERNEL_ADDRESS IA64_RR_BASE(5) +#define VM_MAX_ADDRESS IA64_RR_BASE(5) +#define VM_GATEWAY_SIZE PAGE_SIZE +#define VM_MAXUSER_ADDRESS (VM_MAX_ADDRESS + VM_GATEWAY_SIZE) +#define VM_MIN_KERNEL_ADDRESS VM_MAXUSER_ADDRESS #define VM_MAX_KERNEL_ADDRESS (IA64_RR_BASE(6) - 1) -#define KERNBASE (VM_MIN_KERNEL_ADDRESS) +#define KERNBASE VM_MAX_ADDRESS /* virtual sizes (bytes) for various kernel submaps */ #ifndef VM_KMEM_SIZE diff --git a/sys/kern/kern_kse.c b/sys/kern/kern_kse.c index 4ad0adb..e02b7f2 100644 --- a/sys/kern/kern_kse.c +++ b/sys/kern/kern_kse.c @@ -1221,7 +1221,7 @@ thread_exit(void) } /* XXX Shouldn't cpu_throw() here. */ mtx_assert(&sched_lock, MA_OWNED); -#if defined(__i386__) || defined(__sparc64__) || defined(__amd64__) +#if !defined(__alpha__) && !defined(__powerpc__) cpu_throw(td, choosethread()); #else cpu_throw(); diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index fbf6ed3..b034048 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -460,7 +460,7 @@ mi_switch(void) { struct bintime new_switchtime; struct thread *td; -#if defined(__i386__) || defined(__sparc64__) || defined(__amd64__) +#if !defined(__alpha__) && !defined(__powerpc__) struct thread *newtd; #endif struct proc *p; @@ -518,7 +518,7 @@ mi_switch(void) thread_switchout(td); sched_switchout(td); -#if defined(__i386__) || defined(__sparc64__) || defined(__amd64__) +#if !defined(__alpha__) && !defined(__powerpc__) newtd = choosethread(); if (td != newtd) cpu_switch(td, newtd); /* SHAZAM!! */ diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c index 72cc75c..a7be35f 100644 --- a/sys/kern/kern_thr.c +++ b/sys/kern/kern_thr.c @@ -111,7 +111,7 @@ thr_exit1(void) sched_exit_thread(TAILQ_NEXT(td, td_kglist), td); thread_stash(td); -#if defined(__i386__) || defined(__sparc64__) || defined(__amd64__) +#if !defined(__alpha__) && !defined(__powerpc__) cpu_throw(td, choosethread()); #else cpu_throw(); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 4ad0adb..e02b7f2 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -1221,7 +1221,7 @@ thread_exit(void) } /* XXX Shouldn't cpu_throw() here. */ mtx_assert(&sched_lock, MA_OWNED); -#if defined(__i386__) || defined(__sparc64__) || defined(__amd64__) +#if !defined(__alpha__) && !defined(__powerpc__) cpu_throw(td, choosethread()); #else cpu_throw(); diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 1b7982c..3aac050 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -860,7 +860,7 @@ int sigonstack(size_t sp); void sleepinit(void); void stopevent(struct proc *, u_int, u_int); void cpu_idle(void); -#if defined(__i386__) || defined(__sparc64__) || defined(__amd64__) +#if !defined(__alpha__) && !defined(__powerpc__) void cpu_switch(struct thread *old, struct thread *new); void cpu_throw(struct thread *old, struct thread *new) __dead2; #else diff --git a/usr.bin/truss/ia64-fbsd.c b/usr.bin/truss/ia64-fbsd.c index 7cf4f52..9f065a1 100644 --- a/usr.bin/truss/ia64-fbsd.c +++ b/usr.bin/truss/ia64-fbsd.c @@ -136,14 +136,14 @@ ia64_syscall_entry(struct trussinfo *trussinfo, int nargs) { fprintf(trussinfo->outfile, "-- CANNOT READ REGISTERS --\n"); return; } - parm_offset = regs.r_gr[12] + 16; + parm_offset = regs.r_special.sp + 16; /* * FreeBSD has two special kinds of system call redirctions -- * SYS_syscall, and SYS___syscall. The former is the old syscall() * routine, basicly; the latter is for quad-aligned arguments. */ - syscall_num = regs.r_gr[15]; + syscall_num = regs.r_scratch.gr15; /* XXX double-check. */ switch (syscall_num) { case SYS_syscall: lseek(Procfd, parm_offset, SEEK_SET); @@ -293,8 +293,8 @@ ia64_syscall_exit(struct trussinfo *trussinfo, int syscall_num __unused) { fprintf(trussinfo->outfile, "-- CANNOT READ REGISTERS --\n"); return (-1); } - retval = regs.r_gr[8]; - errorp = (regs.r_gr[10] != 0) ? 1 : 0; + retval = regs.r_scratch.gr8; + errorp = (regs.r_scratch.gr10 != 0) ? 1 : 0; /* * This code, while simpler than the initial versions I used, could |