Demark sections of code that need special fault handling with labels.

Check if the trapped pc is inside of the demarked sections to implement
fault recovery for copyin etc, instead of pcb_onfault.  Handle recovery
from data access exceptions as well as page faults.

Inspired by:	bde's sys.dif

5 files changed, 169 insertions, 148 deletions

diff --git a/sys/sparc64/include/pcb.h b/sys/sparc64/include/pcb.h
index a70f61a..36d978b 100644
--- a/sys/sparc64/include/pcb.h
+++ b/sys/sparc64/include/pcb.h
@@ -39,7 +39,6 @@ struct pcb {
 	struct	fpstate	pcb_fpstate;
 	u_long	pcb_fp;
 	u_long	pcb_pc;
-	caddr_t	pcb_onfault;
 	u_long	pcb_nsaved;
 	u_long	pcb_rwsp[MAXWIN];
 	struct	rwindow pcb_rw[MAXWIN];
diff --git a/sys/sparc64/include/tlb.h b/sys/sparc64/include/tlb.h
index 8c6ce1a..6cffc1c 100644
--- a/sys/sparc64/include/tlb.h
+++ b/sys/sparc64/include/tlb.h
@@ -81,7 +81,10 @@
 #define	MMU_SFSR_FT_SIZE		(6)
 #define	MMU_SFSR_CT_SIZE		(2)
 
-#define	MMU_SFSR_W			(1L << MMU_SFSR_W_SHIFT)
+#define	MMU_SFSR_GET_ASI(sfsr) \
+	(((sfsr) >> MMU_SFSR_ASI_SHIFT) & ((1UL << MMU_SFSR_ASI_SIZE) - 1))
+#define	MMU_SFSR_W			(1UL << MMU_SFSR_W_SHIFT)
+#define	MMU_SFSR_FV			(1UL << MMU_SFSR_FV_SHIFT)
 
 struct tlb_entry;
 
diff --git a/sys/sparc64/sparc64/genassym.c b/sys/sparc64/sparc64/genassym.c
index 57827b6..9d0dfa2 100644
--- a/sys/sparc64/sparc64/genassym.c
+++ b/sys/sparc64/sparc64/genassym.c
@@ -240,8 +240,6 @@ ASSYM(PCB_SIZEOF, sizeof(struct pcb));
 ASSYM(PCB_FPSTATE, offsetof(struct pcb, pcb_fpstate));
 ASSYM(PCB_FP, offsetof(struct pcb, pcb_fp));
 ASSYM(PCB_PC, offsetof(struct pcb, pcb_pc));
-ASSYM(PCB_Y, offsetof(struct pcb, pcb_pc));
-ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
 ASSYM(PCB_NSAVED, offsetof(struct pcb, pcb_nsaved));
 ASSYM(PCB_RWSP, offsetof(struct pcb, pcb_rwsp));
 ASSYM(PCB_RW, offsetof(struct pcb, pcb_rw));
diff --git a/sys/sparc64/sparc64/support.S b/sys/sparc64/sparc64/support.S
index 68cf714..649f0b8 100644
--- a/sys/sparc64/sparc64/support.S
+++ b/sys/sparc64/sparc64/support.S
@@ -211,37 +211,45 @@
 	 inc	dst ; \
 6:
 
-#define	CATCH_SETUP(label) \
-	SET(label, %g2, %g1) ; \
-	stx	%g1, [PCB_REG + PCB_ONFAULT]
-
-#define	CATCH_END() \
-	stx	%g0, [PCB_REG + PCB_ONFAULT]
-
-#define	FU_ALIGNED(loader, label) \
-	CATCH_SETUP(label) ; \
-	loader	[%o0] ASI_AIUP, %o0 ; \
-	retl ; \
-	 CATCH_END()
-
-#define	FU_BYTES(loader, size, label) \
-	btst	(size) - 1, %o0 ; \
-	bnz,pn	%xcc, .Lfsalign ; \
-	 EMPTY ; \
-	FU_ALIGNED(loader, label)
-
-#define	SU_ALIGNED(storer, label) \
-	CATCH_SETUP(label) ; \
-	storer	%o1, [%o0] ASI_AIUP ; \
-	CATCH_END() ; \
-	retl ; \
-	 clr	%o0
+/*
+ * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len)
+ */
+ENTRY(ascopy)
+	wr	%o0, 0, %asi
+	_MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi)
+	retl
+	 nop
+END(ascopy)
 
-#define	SU_BYTES(storer, size, label) \
-	btst	(size) - 1, %o0 ; \
-	bnz,pn	%xcc, .Lfsalign ; \
-	 EMPTY ; \
-	SU_ALIGNED(storer, label)
+/*
+ * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len)
+ */
+ENTRY(ascopyfrom)
+	wr	%o0, 0, %asi
+	_MEMCPY(%o2, %o1, %o3, E, E, a, %asi)
+	retl
+	 nop
+END(ascopyfrom)
+
+/*
+ * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len)
+ */
+ENTRY(ascopyto)
+	wr	%o1, 0, %asi
+	_MEMCPY(%o2, %o0, %o3, a, %asi, E, E)
+	retl
+	 nop
+END(ascopyto)
+
+/*
+ * void aszero(u_long asi, vm_offset_t pa, size_t len)
+ */
+ENTRY(aszero)
+	wr	%o0, 0, %asi
+	_MEMSET(%o1, %g0, %o2, a, %asi)
+	retl
+	 nop
+END(aszero)
 
 /*
  * int bcmp(const void *b1, const void *b2, size_t len)
@@ -308,44 +316,13 @@ ENTRY(bzero)
 END(bzero)
 
 /*
- * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len)
- */
-ENTRY(ascopy)
-	wr	%o0, 0, %asi
-	_MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi)
-	retl
-	 nop
-END(ascopy)
-
-/*
- * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len)
- */
-ENTRY(ascopyfrom)
-	wr	%o0, 0, %asi
-	_MEMCPY(%o2, %o1, %o3, E, E, a, %asi)
-	retl
-	 nop
-END(ascopyfrom)
-
-/*
- * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len)
- */
-ENTRY(ascopyto)
-	wr	%o1, 0, %asi
-	_MEMCPY(%o2, %o0, %o3, a, %asi, E, E)
-	retl
-	 nop
-END(ascopyto)
-
-/*
- * void aszero(u_long asi, vm_offset_t pa, size_t len)
+ * int copystr(const void *src, void *dst, size_t len, size_t *done)
  */
-ENTRY(aszero)
-	wr	%o0, 0, %asi
-	_MEMSET(%o1, %g0, %o2, a, %asi)
+ENTRY(copystr)
+	_COPYSTR(%o0, %o1, %o2, %o3, E, E, E, E)
 	retl
-	 nop
-END(aszero)
+	 mov	%g1, %o0
+END(copystr)
 
 /*
  * void *memcpy(void *dst, const void *src, size_t len)
@@ -367,14 +344,16 @@ ENTRY(memset)
 	 nop
 END(memset)
 
+	.globl	copy_nofault_begin
+copy_nofault_begin:
+	nop
+
 /*
  * int copyin(const void *uaddr, void *kaddr, size_t len)
  */
 ENTRY(copyin)
-	CATCH_SETUP(.Lefault)
 	wr	%g0, ASI_AIUP, %asi
 	_MEMCPY(%o1, %o0, %o2, E, E, a, %asi)
-	CATCH_END()
 	retl
 	 clr	%o0
 END(copyin)
@@ -383,10 +362,8 @@ END(copyin)
  * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
  */
 ENTRY(copyinstr)
-	CATCH_SETUP(.Lefault)
 	wr	%g0, ASI_AIUP, %asi
 	_COPYSTR(%o0, %o1, %o2, %o3, a, %asi, E, E)
-	CATCH_END()
 	retl
 	 mov	%g1, %o0
 END(copyinstr)
@@ -395,128 +372,137 @@ END(copyinstr)
  * int copyout(const void *kaddr, void *uaddr, size_t len)
  */
 ENTRY(copyout)
-	CATCH_SETUP(.Lefault)
 	wr	%g0, ASI_AIUP, %asi
 	_MEMCPY(%o1, %o0, %o2, a, %asi, E, E)
-	CATCH_END()
 	retl
 	 clr	%o0
 END(copyout)
 
-.Lefault:
-	CATCH_END()
-	retl
-	 mov	EFAULT, %o0
+	.globl	copy_nofault_end
+copy_nofault_end:
+	nop
 
-/*
- * int copystr(const void *src, void *dst, size_t len, size_t *done)
- */
-ENTRY(copystr)
-	_COPYSTR(%o0, %o1, %o2, %o3, E, E, E, E)
+ENTRY(copy_fault)
 	retl
-	 mov	%g1, %o0
-END(copystr)
+	 mov	EFAULT, %o0
+END(copy_fault)
 
-/*
- * int fubyte(const void *base)
- */
-ENTRY(fubyte)
-	FU_ALIGNED(lduba, .Lfsfault)
-END(fubyte)
+	.globl	fs_nofault_begin
+fs_nofault_begin:
+	nop
 
 /*
- * long fuword(const void *base)
+ * Chatty aliases for fetch, store functions.
  */
-ENTRY(fuword)
-	FU_BYTES(ldxa, 8, .Lfsfault)
-END(fuword)
+	.globl	fubyte, fusword, fuword, subyte, susword, suword
+	.set	fubyte, fuword8
+	.set	fusword, fuword16
+	.set	fuword, fuword64
+	.set	subyte, suword8
+	.set	susword, suword16
+	.set	suword, suword64
 
 /*
- * int fuswintr(const void *base)
+ * int fuword8(const void *base)
  */
-ENTRY(fuswintr)
-	FU_BYTES(lduha, 2, fsbail)
-END(fuswintr)
+ENTRY(fuword8)
+	retl
+	 lduba	[%o0] ASI_AIUP, %o0
+END(fuword8)
 
 /*
- * int16_t fuword16(const void *base)
+ * int fuword16(const void *base)
  */
 ENTRY(fuword16)
-	FU_BYTES(lduha, 2, .Lfsfault)
+	retl
+	 lduha	[%o0] ASI_AIUP, %o0
 END(fuword16)
 
 /*
  * int32_t fuword32(const void *base)
  */
 ENTRY(fuword32)
-	FU_BYTES(lduwa, 4, .Lfsfault)
+	retl
+	 lduwa	[%o0] ASI_AIUP, %o0
 END(fuword32)
 
 /*
  * int64_t fuword64(const void *base)
  */
 ENTRY(fuword64)
-	FU_BYTES(ldxa, 8, .Lfsfault)
+	retl
+	 ldxa	[%o0] ASI_AIUP, %o0
 END(fuword64)
 
 /*
- * int subyte(const void *base, int byte)
+ * int suword8(const void *base, int word)
  */
-ENTRY(subyte)
-	SU_ALIGNED(stba, .Lfsfault)
-END(subyte)
-
-/*
- * int suword(const void *base, long word)
- */
-ENTRY(suword)
-	SU_BYTES(stxa, 8, .Lfsfault)
-END(suword)
-
-/*
- * int suswintr(const void *base, int word)
- */
-ENTRY(suswintr)
-	SU_BYTES(stwa, 2, fsbail)
-END(suswintr)
+ENTRY(suword8)
+	stba	%o1, [%o0] ASI_AIUP
+	retl
+	 clr	%o0
+END(suword8)
 
 /*
- * int suword16(const void *base, int16_t word)
+ * int suword16(const void *base, int word)
  */
 ENTRY(suword16)
-	SU_BYTES(stha, 2, .Lfsfault)
+	stha	%o1, [%o0] ASI_AIUP
+	retl
+	 clr	%o0
 END(suword16)
 
 /*
  * int suword32(const void *base, int32_t word)
  */
 ENTRY(suword32)
-	SU_BYTES(stwa, 4, .Lfsfault)
+	stwa	%o1, [%o0] ASI_AIUP
+	retl
+	 clr	%o0
 END(suword32)
 
 /*
  * int suword64(const void *base, int64_t word)
  */
 ENTRY(suword64)
-	SU_BYTES(stxa, 8, .Lfsfault)
+	stxa	%o1, [%o0] ASI_AIUP
+	retl
+	 clr	%o0
 END(suword64)
 
-	_ALIGN_TEXT
-.Lfsalign:
+	.globl	fs_nofault_intr_begin
+fs_nofault_intr_begin:
+	nop
+
+/*
+ * int fuswintr(const void *base)
+ */
+ENTRY(fuswintr)
 	retl
-	 mov	-1, %o0
+	 lduha	[%o0] ASI_AIUP, %o0
+END(fuswintr)
 
-	_ALIGN_TEXT
-.Lfsfault:
-	CATCH_END()
+/*
+ * int suswintr(const void *base, int word)
+ */
+ENTRY(suswintr)
+	stha	%o1, [%o0] ASI_AIUP
 	retl
-	 mov	-1, %o0
+	 clr	%o0
+END(suswintr)
+
+	.globl	fs_nofault_intr_end
+fs_nofault_intr_end:
+	nop
+
+	.globl	fs_nofault_end
+fs_nofault_end:
+	nop
 
-ENTRY(fsbail)
-	CATCH_END()
+ENTRY(fs_fault)
 	retl
 	 mov	-1, %o0
-END(fsbail)
+END(fsfault)
 
 ENTRY(longjmp)
 	set	1, %g3
diff --git a/sys/sparc64/sparc64/trap.c b/sys/sparc64/sparc64/trap.c
index 0ab35af..65eda04 100644
--- a/sys/sparc64/sparc64/trap.c
+++ b/sys/sparc64/sparc64/trap.c
@@ -91,7 +91,15 @@ void syscall(struct trapframe *tf);
 
 static int trap_pfault(struct thread *td, struct trapframe *tf);
 
-extern char fsbail[];
+extern char copy_fault[];
+extern char copy_nofault_begin[];
+extern char copy_nofault_end[];
+
+extern char fs_fault[];
+extern char fs_nofault_begin[];
+extern char fs_nofault_end[];
+extern char fs_nofault_intr_begin[];
+extern char fs_nofault_intr_end[];
 
 extern char *syscallnames[];
 
@@ -303,6 +311,24 @@ if ((type & ~T_KERNEL) != T_BREAKPOINT)
 			goto out;
 		break;
 #endif
+	case T_DATA_EXCEPTION | T_KERNEL:
+	case T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL:
+		if ((tf->tf_sfsr & MMU_SFSR_FV) == 0 ||
+		    MMU_SFSR_GET_ASI(tf->tf_sfsr) != ASI_AIUP)
+			break;
+		if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
+		    tf->tf_tpc <= (u_long)copy_nofault_end) {
+			tf->tf_tpc = (u_long)copy_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			goto out;
+		}
+		if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
+		    tf->tf_tpc <= (u_long)fs_nofault_end) {
+			tf->tf_tpc = (u_long)fs_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			goto out;
+		}
+		break;
 	case T_DATA_MISS | T_KERNEL:
 	case T_DATA_PROTECTION | T_KERNEL:
 	case T_INSTRUCTION_MISS | T_KERNEL:
@@ -439,9 +465,12 @@ trap_pfault(struct thread *td, struct trapframe *tf)
 
 	if (ctx != TLB_CTX_KERNEL) {
 		if ((tf->tf_tstate & TSTATE_PRIV) != 0 &&
-		    (td->td_intr_nesting_level != 0 ||
-		    pcb->pcb_onfault == NULL || pcb->pcb_onfault == fsbail))
-			return (-1);
+		    (tf->tf_tpc >= (u_long)fs_nofault_intr_begin &&
+		     tf->tf_tpc <= (u_long)fs_nofault_intr_end)) {
+			tf->tf_tpc = (u_long)fs_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			return (0);
+		}
 
 		/*
 		 * This is a fault on non-kernel virtual memory.
@@ -467,9 +496,9 @@ trap_pfault(struct thread *td, struct trapframe *tf)
 		PROC_UNLOCK(p);
 	} else {
 		/*
-		 * This is a fault on kernel virtual memory.  Attempts to access
-		 * kernel memory from user mode cause privileged action traps,
-		 * not page fault.
+		 * This is a fault on kernel virtual memory.  Attempts to
+		 * access kernel memory from user mode cause privileged
+		 * action traps, not page fault.
 		 */
 		KASSERT(tf->tf_tstate & TSTATE_PRIV,
 		    ("trap_pfault: fault on nucleus context from user mode"));
@@ -485,10 +514,16 @@ trap_pfault(struct thread *td, struct trapframe *tf)
 	    td, va, rv);
 	if (rv == KERN_SUCCESS)
 		return (0);
-	if ((tf->tf_tstate & TSTATE_PRIV) != 0) {
-		if (td->td_intr_nesting_level == 0 &&
-		    pcb->pcb_onfault != NULL) {
-			tf->tf_tpc = (u_long)pcb->pcb_onfault;
+	if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) {
+		if (tf->tf_tpc >= (u_long)fs_nofault_begin &&
+		    tf->tf_tpc <= (u_long)fs_nofault_end) {
+			tf->tf_tpc = (u_long)fs_fault;
+			tf->tf_tnpc = tf->tf_tpc + 4;
+			return (0);
+		}
+		if (tf->tf_tpc >= (u_long)copy_nofault_begin &&
+		    tf->tf_tpc <= (u_long)copy_nofault_end) {
+			tf->tf_tpc = (u_long)copy_fault;
 			tf->tf_tnpc = tf->tf_tpc + 4;
 			return (0);
 		}