Fix fu{byte|word*} and su{byte|word*}:

o  If the address was not within user space we jumped to fusufault
   where we would clear pcb_onfault and return 0. There are two
   bugs here:
   1. We never got to the point where we assigned the address of
      pcb_onfault to r15, which means that we would clobber some
      random memory location, including I/O space or ROM.
   2. We're supposed to return -1 on error.
o  Make sure we have proper memory ordering for setting pcb_onfault,
   doing the memory access to user space and clearing pcb_onfault.
   For the fu* family of functions this means that we need a mf
   instruction, because we don't have acquire semantics on stores
   and release semantics on loads (hence st;ld cannot be ordered
   without intermediate mf).

While here, implement casuptr() so that we are a (small) step
closer to supporting libthr and deobfuscate the non-implementation
of {f|s}uswintr.

Approved by: re@ (blanket)

3 files changed, 902 insertions, 298 deletions

diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index 7386ce3..65e98ff 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -1366,9 +1366,3 @@ ia64_pack_bundle(u_int64_t *lowp, u_int64_t *highp,
 	*lowp = low;
 	*highp = high;
 }
-
-intptr_t
-casuptr(intptr_t *p, intptr_t old, intptr_t new)
-{
-	return (-1);
-}
diff --git a/sys/ia64/ia64/support.S b/sys/ia64/ia64/support.S
index 7403a01..252ab23 100644
--- a/sys/ia64/ia64/support.S
+++ b/sys/ia64/ia64/support.S
@@ -181,179 +181,484 @@ ENTRY(ia64_call_efi_physical, 6)
 END(ia64_call_efi_physical)
 	
 /**************************************************************************/
-	
-/*
- * fu{byte,word} : fetch a byte (word) from user memory
- */
-	
-ENTRY(suword64, 2)
-XENTRY(suword)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	st8.rel	[in0]=in1			// try the store
+ENTRY(fusufault, 0)
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault.
+	add		ret0=-1,r0
+	br.ret.sptk	rp
 	;;
-	st8	[r15]=r0			// clean up
+}
+END(fusufault)
 
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(suword64)
-	
-ENTRY(suword32, 2)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+/*
+ * casuptr(intptr_t *p, intptr_t old, intptr_t new)
+ *	Perform a compare-exchange in user space.
+ */
+ENTRY(casuptr, 3)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	mov		ar.ccv=in1
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	cmpxchg8.rel	ret0=[in0],in2,ar.ccv
+	nop		0
+	;;
+}
+{	.mfb
+	st8.rel		[r15]=r0		// Clear onfault
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(casuptr)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	st4.rel	[in0]=in1			// try the store
-	;;
-	st8	[r15]=r0			// clean up
+/*
+ * subyte(void *addr, int byte)
+ * suword16(void *addr, int word)
+ * suword32(void *addr, int word)
+ * suword64|suword(void *addr, long word)
+ *	Store in user space
+ */
 
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(suword32)
-	
 ENTRY(subyte, 2)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	st1.rel		[in0]=in1
+	nop		0
+	;;
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(subyte)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
+ENTRY(suword16, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
 	;;
-	ld8	r15=[r15]
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
 	;;
-	add	r15=PCB_ONFAULT,r15
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
 	;;
-	st8	[r15]=r14
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
 	;;
-	st1.rel	[in0]=in1			// try the store
+	nop		0
+	add		r15=PCB_ONFAULT,r15
 	;;
-	st8	[r15]=r0			// clean up
-
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(subyte)
-
-ENTRY(fuword64, 1)
-XENTRY(fuword)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
-
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
 	;;
-	ld8	r15=[r15]
+	st2.rel		[in0]=in1
+	nop		0
 	;;
-	add	r15=TD_PCB,r15			// find pcb
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
 	;;
-	ld8	r15=[r15]
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	ld8.acq	ret0=[in0]			// try the fetch
-	;;
-	st8	[r15]=r0			// clean up
+}
+END(suword16)
 
-	br.ret.sptk.few rp
-END(fuword64)
+ENTRY(suword32, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	st4.rel		[in0]=in1
+	nop		0
+	;;
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(suword32)
 
-ENTRY(fuword32, 1)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+ENTRY(suword64, 2)
+XENTRY(suword)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	st8.rel		[in0]=in1
+	nop		0
+	;;
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(suword64)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	ld4.acq	ret0=[in0]			// try the fetch
-	;;
-	st8	[r15]=r0			// clean up
+/*
+ * fubyte(void *addr, int byte)
+ * fuword16(void *addr, int word)
+ * fuword32(void *addr, int word)
+ * fuword64|fuword(void *addr, long word)
+ *	Fetch from user space
+ */
 
-	br.ret.sptk.few rp
+ENTRY(fubyte, 1)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld1		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(fubyte)
+
+ENTRY(fuword16, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld2		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(fuword16)
+
+ENTRY(fuword32, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld4		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
 END(fuword32)
 
-ENTRY(fubyte, 1)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+ENTRY(fuword64, 2)
+XENTRY(fuword)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld8		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(fuword64)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
+/*
+ * fuswintr(void *addr)
+ * suswintr(void *addr)
+ */
+
+ENTRY(fswintrberr, 0)
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	st8	[r15]=r14
+}
+END(fswintrberr)
+
+ENTRY(fuswintr, 1)
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	ld1.acq	ret0=[in0]			// try the fetch
+}
+END(fuswintr)
+
+ENTRY(suswintr, 0)
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	st8	[r15]=r0			// clean up
+}
+END(suswintr)
 
-	br.ret.sptk.few rp
-END(fubyte)
-	
-ENTRY(fusufault, 0)
-	st8	[r15]=r0 ;;			// r15 points at onfault
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(fusufault)
-	
-ENTRY(fswintrberr, 0)
-XENTRY(fuswintr)			/* XXX 16 bit short for profiling */
-XENTRY(suswintr)			/* XXX 16 bit short for profiling */
-	mov	ret0=-1
-	br.ret.sptk.few rp
-END(fswintrberr)
-	
 /**************************************************************************/
 
 /*
diff --git a/sys/ia64/ia64/support.s b/sys/ia64/ia64/support.s
index 7403a01..252ab23 100644
--- a/sys/ia64/ia64/support.s
+++ b/sys/ia64/ia64/support.s
@@ -181,179 +181,484 @@ ENTRY(ia64_call_efi_physical, 6)
 END(ia64_call_efi_physical)
 	
 /**************************************************************************/
-	
-/*
- * fu{byte,word} : fetch a byte (word) from user memory
- */
-	
-ENTRY(suword64, 2)
-XENTRY(suword)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	st8.rel	[in0]=in1			// try the store
+ENTRY(fusufault, 0)
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault.
+	add		ret0=-1,r0
+	br.ret.sptk	rp
 	;;
-	st8	[r15]=r0			// clean up
+}
+END(fusufault)
 
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(suword64)
-	
-ENTRY(suword32, 2)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+/*
+ * casuptr(intptr_t *p, intptr_t old, intptr_t new)
+ *	Perform a compare-exchange in user space.
+ */
+ENTRY(casuptr, 3)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	mov		ar.ccv=in1
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	cmpxchg8.rel	ret0=[in0],in2,ar.ccv
+	nop		0
+	;;
+}
+{	.mfb
+	st8.rel		[r15]=r0		// Clear onfault
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(casuptr)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	st4.rel	[in0]=in1			// try the store
-	;;
-	st8	[r15]=r0			// clean up
+/*
+ * subyte(void *addr, int byte)
+ * suword16(void *addr, int word)
+ * suword32(void *addr, int word)
+ * suword64|suword(void *addr, long word)
+ *	Store in user space
+ */
 
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(suword32)
-	
 ENTRY(subyte, 2)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	st1.rel		[in0]=in1
+	nop		0
+	;;
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(subyte)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
+ENTRY(suword16, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
 	;;
-	ld8	r15=[r15]
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
 	;;
-	add	r15=PCB_ONFAULT,r15
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
 	;;
-	st8	[r15]=r14
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
 	;;
-	st1.rel	[in0]=in1			// try the store
+	nop		0
+	add		r15=PCB_ONFAULT,r15
 	;;
-	st8	[r15]=r0			// clean up
-
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(subyte)
-
-ENTRY(fuword64, 1)
-XENTRY(fuword)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
-
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
 	;;
-	ld8	r15=[r15]
+	st2.rel		[in0]=in1
+	nop		0
 	;;
-	add	r15=TD_PCB,r15			// find pcb
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
 	;;
-	ld8	r15=[r15]
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	ld8.acq	ret0=[in0]			// try the fetch
-	;;
-	st8	[r15]=r0			// clean up
+}
+END(suword16)
 
-	br.ret.sptk.few rp
-END(fuword64)
+ENTRY(suword32, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	st4.rel		[in0]=in1
+	nop		0
+	;;
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(suword32)
 
-ENTRY(fuword32, 1)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+ENTRY(suword64, 2)
+XENTRY(suword)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	st8.rel		[in0]=in1
+	nop		0
+	;;
+}
+{	.mib
+	st8.rel		[r15]=r0		// Clear onfault
+	mov		ret0=r0
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(suword64)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
-	;;
-	st8	[r15]=r14
-	;;
-	ld4.acq	ret0=[in0]			// try the fetch
-	;;
-	st8	[r15]=r0			// clean up
+/*
+ * fubyte(void *addr, int byte)
+ * fuword16(void *addr, int word)
+ * fuword32(void *addr, int word)
+ * fuword64|fuword(void *addr, long word)
+ *	Fetch from user space
+ */
 
-	br.ret.sptk.few rp
+ENTRY(fubyte, 1)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld1		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(fubyte)
+
+ENTRY(fuword16, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld2		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(fuword16)
+
+ENTRY(fuword32, 2)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld4		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
 END(fuword32)
 
-ENTRY(fubyte, 1)
-	movl	r14=VM_MAX_ADDRESS;;		// make sure address is ok
-	cmp.geu	p6,p0=in0,r14
-(p6)	br.dpnt.few fusufault
+ENTRY(fuword64, 2)
+XENTRY(fuword)
+{	.mlx
+	add		r15=PC_CURTHREAD,r13
+	movl		r14=VM_MAX_ADDRESS
+	;;
+}
+{	.mib
+	ld8		r15=[r15]		// r15 = curthread
+	cmp.geu		p6,p0=in0,r14
+(p6)	br.dpnt.few	1f
+	;;
+}
+{	.mlx
+	add		r15=TD_PCB,r15
+	movl		r14=fusufault
+	;;
+}
+{	.mmi
+	ld8		r15=[r15]		// r15 = PCB
+	;;
+	nop		0
+	add		r15=PCB_ONFAULT,r15
+	;;
+}
+{	.mmi
+	st8		[r15]=r14		// Set onfault
+	;;
+	mf
+	nop		0
+	;;
+}
+{	.mmb
+	ld8		ret0=[in0]
+	st8.rel		[r15]=r0		// Clear onfault
+	br.ret.sptk	rp
+	;;
+}
+1:
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
+	;;
+}
+END(fuword64)
 
-	movl	r14=fusufault			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=TD_PCB,r15			// find pcb
-	;;
-	ld8	r15=[r15]
-	;;
-	add	r15=PCB_ONFAULT,r15
+/*
+ * fuswintr(void *addr)
+ * suswintr(void *addr)
+ */
+
+ENTRY(fswintrberr, 0)
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	st8	[r15]=r14
+}
+END(fswintrberr)
+
+ENTRY(fuswintr, 1)
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	ld1.acq	ret0=[in0]			// try the fetch
+}
+END(fuswintr)
+
+ENTRY(suswintr, 0)
+{	.mfb
+	add		ret0=-1,r0
+	nop		0
+	br.ret.sptk	rp
 	;;
-	st8	[r15]=r0			// clean up
+}
+END(suswintr)
 
-	br.ret.sptk.few rp
-END(fubyte)
-	
-ENTRY(fusufault, 0)
-	st8	[r15]=r0 ;;			// r15 points at onfault
-	mov	ret0=r0
-	br.ret.sptk.few rp
-END(fusufault)
-	
-ENTRY(fswintrberr, 0)
-XENTRY(fuswintr)			/* XXX 16 bit short for profiling */
-XENTRY(suswintr)			/* XXX 16 bit short for profiling */
-	mov	ret0=-1
-	br.ret.sptk.few rp
-END(fswintrberr)
-	
 /**************************************************************************/
 
 /*