MIPS: lib: csum_partial: Add macro to build csum_partial symbols

In preparation for EVA support, we use a macro to build the
__csum_partial_copy_user main code so it can be shared across
multiple implementations. EVA uses the same code but it replaces
the load/store/prefetch instructions with the EVA specific ones
therefore using a macro avoids unnecessary code duplications.

Signed-off-by: Markos Chandras <markos.chandras@imgtec.com>

1 files changed, 108 insertions, 92 deletions

diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S
index bff5167..62c8768 100644
--- a/arch/mips/lib/csum_partial.S
+++ b/arch/mips/lib/csum_partial.S
@@ -331,6 +331,10 @@ LEAF(csum_partial)
 /* Instruction type */
 #define LD_INSN 1
 #define ST_INSN 2
+#define LEGACY_MODE 1
+#define EVA_MODE    2
+#define USEROP   1
+#define KERNELOP 2
 
 /*
  * Wrapper to add an entry in the exception table
@@ -343,10 +347,12 @@ LEAF(csum_partial)
  * handler : Exception handler
  */
 #define EXC(insn, type, reg, addr, handler)	\
-9:	insn reg, addr;				\
-	.section __ex_table,"a";		\
-	PTR	9b, handler;			\
-	.previous
+	.if \mode == LEGACY_MODE;		\
+9:		insn reg, addr;			\
+		.section __ex_table,"a";	\
+		PTR	9b, handler;		\
+		.previous;			\
+	.endif
 
 #undef LOAD
 
@@ -419,16 +425,20 @@ LEAF(csum_partial)
 	.set	at=v1
 #endif
 
-LEAF(__csum_partial_copy_kernel)
-FEXPORT(__csum_partial_copy_to_user)
-FEXPORT(__csum_partial_copy_from_user)
+	.macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
+
 	PTR_ADDU	AT, src, len	/* See (1) above. */
+	/* initialize __nocheck if this the first time we execute this
+	 * macro
+	 */
 #ifdef CONFIG_64BIT
 	move	errptr, a4
 #else
 	lw	errptr, 16(sp)
 #endif
-FEXPORT(csum_partial_copy_nocheck)
+	.if \__nocheck == 1
+	FEXPORT(csum_partial_copy_nocheck)
+	.endif
 	move	sum, zero
 	move	odd, zero
 	/*
@@ -444,48 +454,48 @@ FEXPORT(csum_partial_copy_nocheck)
 	 */
 	sltu	t2, len, NBYTES
 	and	t1, dst, ADDRMASK
-	bnez	t2, .Lcopy_bytes_checklen
+	bnez	t2, .Lcopy_bytes_checklen\@
 	 and	t0, src, ADDRMASK
 	andi	odd, dst, 0x1			/* odd buffer? */
-	bnez	t1, .Ldst_unaligned
+	bnez	t1, .Ldst_unaligned\@
 	 nop
-	bnez	t0, .Lsrc_unaligned_dst_aligned
+	bnez	t0, .Lsrc_unaligned_dst_aligned\@
 	/*
 	 * use delay slot for fall-through
 	 * src and dst are aligned; need to compute rem
 	 */
-.Lboth_aligned:
+.Lboth_aligned\@:
 	 SRL	t0, len, LOG_NBYTES+3	 # +3 for 8 units/iter
-	beqz	t0, .Lcleanup_both_aligned # len < 8*NBYTES
+	beqz	t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
 	 nop
 	SUB	len, 8*NBYTES		# subtract here for bgez loop
 	.align	4
 1:
-	LOAD(t0, UNIT(0)(src), .Ll_exc)
-	LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
-	LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
-	LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
-	LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
-	LOAD(t5, UNIT(5)(src), .Ll_exc_copy)
-	LOAD(t6, UNIT(6)(src), .Ll_exc_copy)
-	LOAD(t7, UNIT(7)(src), .Ll_exc_copy)
+	LOAD(t0, UNIT(0)(src), .Ll_exc\@)
+	LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
+	LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
+	LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
+	LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
+	LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
+	LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
+	LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
 	SUB	len, len, 8*NBYTES
 	ADD	src, src, 8*NBYTES
-	STORE(t0, UNIT(0)(dst),	.Ls_exc)
+	STORE(t0, UNIT(0)(dst),	.Ls_exc\@)
 	ADDC(sum, t0)
-	STORE(t1, UNIT(1)(dst),	.Ls_exc)
+	STORE(t1, UNIT(1)(dst),	.Ls_exc\@)
 	ADDC(sum, t1)
-	STORE(t2, UNIT(2)(dst),	.Ls_exc)
+	STORE(t2, UNIT(2)(dst),	.Ls_exc\@)
 	ADDC(sum, t2)
-	STORE(t3, UNIT(3)(dst),	.Ls_exc)
+	STORE(t3, UNIT(3)(dst),	.Ls_exc\@)
 	ADDC(sum, t3)
-	STORE(t4, UNIT(4)(dst),	.Ls_exc)
+	STORE(t4, UNIT(4)(dst),	.Ls_exc\@)
 	ADDC(sum, t4)
-	STORE(t5, UNIT(5)(dst),	.Ls_exc)
+	STORE(t5, UNIT(5)(dst),	.Ls_exc\@)
 	ADDC(sum, t5)
-	STORE(t6, UNIT(6)(dst),	.Ls_exc)
+	STORE(t6, UNIT(6)(dst),	.Ls_exc\@)
 	ADDC(sum, t6)
-	STORE(t7, UNIT(7)(dst),	.Ls_exc)
+	STORE(t7, UNIT(7)(dst),	.Ls_exc\@)
 	ADDC(sum, t7)
 	.set	reorder				/* DADDI_WAR */
 	ADD	dst, dst, 8*NBYTES
@@ -496,44 +506,44 @@ FEXPORT(csum_partial_copy_nocheck)
 	/*
 	 * len == the number of bytes left to copy < 8*NBYTES
 	 */
-.Lcleanup_both_aligned:
+.Lcleanup_both_aligned\@:
 #define rem t7
-	beqz	len, .Ldone
+	beqz	len, .Ldone\@
 	 sltu	t0, len, 4*NBYTES
-	bnez	t0, .Lless_than_4units
+	bnez	t0, .Lless_than_4units\@
 	 and	rem, len, (NBYTES-1)	# rem = len % NBYTES
 	/*
 	 * len >= 4*NBYTES
 	 */
-	LOAD(t0, UNIT(0)(src), .Ll_exc)
-	LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
-	LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
-	LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
+	LOAD(t0, UNIT(0)(src), .Ll_exc\@)
+	LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
+	LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
+	LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
 	SUB	len, len, 4*NBYTES
 	ADD	src, src, 4*NBYTES
-	STORE(t0, UNIT(0)(dst),	.Ls_exc)
+	STORE(t0, UNIT(0)(dst),	.Ls_exc\@)
 	ADDC(sum, t0)
-	STORE(t1, UNIT(1)(dst),	.Ls_exc)
+	STORE(t1, UNIT(1)(dst),	.Ls_exc\@)
 	ADDC(sum, t1)
-	STORE(t2, UNIT(2)(dst),	.Ls_exc)
+	STORE(t2, UNIT(2)(dst),	.Ls_exc\@)
 	ADDC(sum, t2)
-	STORE(t3, UNIT(3)(dst),	.Ls_exc)
+	STORE(t3, UNIT(3)(dst),	.Ls_exc\@)
 	ADDC(sum, t3)
 	.set	reorder				/* DADDI_WAR */
 	ADD	dst, dst, 4*NBYTES
-	beqz	len, .Ldone
+	beqz	len, .Ldone\@
 	.set	noreorder
-.Lless_than_4units:
+.Lless_than_4units\@:
 	/*
 	 * rem = len % NBYTES
 	 */
-	beq	rem, len, .Lcopy_bytes
+	beq	rem, len, .Lcopy_bytes\@
 	 nop
 1:
-	LOAD(t0, 0(src), .Ll_exc)
+	LOAD(t0, 0(src), .Ll_exc\@)
 	ADD	src, src, NBYTES
 	SUB	len, len, NBYTES
-	STORE(t0, 0(dst), .Ls_exc)
+	STORE(t0, 0(dst), .Ls_exc\@)
 	ADDC(sum, t0)
 	.set	reorder				/* DADDI_WAR */
 	ADD	dst, dst, NBYTES
@@ -552,20 +562,20 @@ FEXPORT(csum_partial_copy_nocheck)
 	 * more instruction-level parallelism.
 	 */
 #define bits t2
-	beqz	len, .Ldone
+	beqz	len, .Ldone\@
 	 ADD	t1, dst, len	# t1 is just past last byte of dst
 	li	bits, 8*NBYTES
 	SLL	rem, len, 3	# rem = number of bits to keep
-	LOAD(t0, 0(src), .Ll_exc)
+	LOAD(t0, 0(src), .Ll_exc\@)
 	SUB	bits, bits, rem # bits = number of bits to discard
 	SHIFT_DISCARD t0, t0, bits
-	STREST(t0, -1(t1), .Ls_exc)
+	STREST(t0, -1(t1), .Ls_exc\@)
 	SHIFT_DISCARD_REVERT t0, t0, bits
 	.set reorder
 	ADDC(sum, t0)
-	b	.Ldone
+	b	.Ldone\@
 	.set noreorder
-.Ldst_unaligned:
+.Ldst_unaligned\@:
 	/*
 	 * dst is unaligned
 	 * t0 = src & ADDRMASK
@@ -576,25 +586,25 @@ FEXPORT(csum_partial_copy_nocheck)
 	 * Set match = (src and dst have same alignment)
 	 */
 #define match rem
-	LDFIRST(t3, FIRST(0)(src), .Ll_exc)
+	LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
 	ADD	t2, zero, NBYTES
-	LDREST(t3, REST(0)(src), .Ll_exc_copy)
+	LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
 	SUB	t2, t2, t1	# t2 = number of bytes copied
 	xor	match, t0, t1
-	STFIRST(t3, FIRST(0)(dst), .Ls_exc)
+	STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
 	SLL	t4, t1, 3		# t4 = number of bits to discard
 	SHIFT_DISCARD t3, t3, t4
 	/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
 	ADDC(sum, t3)
-	beq	len, t2, .Ldone
+	beq	len, t2, .Ldone\@
 	 SUB	len, len, t2
 	ADD	dst, dst, t2
-	beqz	match, .Lboth_aligned
+	beqz	match, .Lboth_aligned\@
 	 ADD	src, src, t2
 
-.Lsrc_unaligned_dst_aligned:
+.Lsrc_unaligned_dst_aligned\@:
 	SRL	t0, len, LOG_NBYTES+2	 # +2 for 4 units/iter
-	beqz	t0, .Lcleanup_src_unaligned
+	beqz	t0, .Lcleanup_src_unaligned\@
 	 and	rem, len, (4*NBYTES-1)	 # rem = len % 4*NBYTES
 1:
 /*
@@ -603,53 +613,53 @@ FEXPORT(csum_partial_copy_nocheck)
  * It's OK to load FIRST(N+1) before REST(N) because the two addresses
  * are to the same unit (unless src is aligned, but it's not).
  */
-	LDFIRST(t0, FIRST(0)(src), .Ll_exc)
-	LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
+	LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
+	LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
 	SUB	len, len, 4*NBYTES
-	LDREST(t0, REST(0)(src), .Ll_exc_copy)
-	LDREST(t1, REST(1)(src), .Ll_exc_copy)
-	LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
-	LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
-	LDREST(t2, REST(2)(src), .Ll_exc_copy)
-	LDREST(t3, REST(3)(src), .Ll_exc_copy)
+	LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
+	LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
+	LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
+	LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
+	LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
+	LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
 	ADD	src, src, 4*NBYTES
 #ifdef CONFIG_CPU_SB1
 	nop				# improves slotting
 #endif
-	STORE(t0, UNIT(0)(dst),	.Ls_exc)
+	STORE(t0, UNIT(0)(dst),	.Ls_exc\@)
 	ADDC(sum, t0)
-	STORE(t1, UNIT(1)(dst),	.Ls_exc)
+	STORE(t1, UNIT(1)(dst),	.Ls_exc\@)
 	ADDC(sum, t1)
-	STORE(t2, UNIT(2)(dst),	.Ls_exc)
+	STORE(t2, UNIT(2)(dst),	.Ls_exc\@)
 	ADDC(sum, t2)
-	STORE(t3, UNIT(3)(dst),	.Ls_exc)
+	STORE(t3, UNIT(3)(dst),	.Ls_exc\@)
 	ADDC(sum, t3)
 	.set	reorder				/* DADDI_WAR */
 	ADD	dst, dst, 4*NBYTES
 	bne	len, rem, 1b
 	.set	noreorder
 
-.Lcleanup_src_unaligned:
-	beqz	len, .Ldone
+.Lcleanup_src_unaligned\@:
+	beqz	len, .Ldone\@
 	 and	rem, len, NBYTES-1  # rem = len % NBYTES
-	beq	rem, len, .Lcopy_bytes
+	beq	rem, len, .Lcopy_bytes\@
 	 nop
 1:
-	LDFIRST(t0, FIRST(0)(src), .Ll_exc)
-	LDREST(t0, REST(0)(src), .Ll_exc_copy)
+	LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
+	LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
 	ADD	src, src, NBYTES
 	SUB	len, len, NBYTES
-	STORE(t0, 0(dst), .Ls_exc)
+	STORE(t0, 0(dst), .Ls_exc\@)
 	ADDC(sum, t0)
 	.set	reorder				/* DADDI_WAR */
 	ADD	dst, dst, NBYTES
 	bne	len, rem, 1b
 	.set	noreorder
 
-.Lcopy_bytes_checklen:
-	beqz	len, .Ldone
+.Lcopy_bytes_checklen\@:
+	beqz	len, .Ldone\@
 	 nop
-.Lcopy_bytes:
+.Lcopy_bytes\@:
 	/* 0 < len < NBYTES  */
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
 #define SHIFT_START 0
@@ -662,12 +672,12 @@ FEXPORT(csum_partial_copy_nocheck)
 	li	t3, SHIFT_START # shift
 /* use .Ll_exc_copy here to return correct sum on fault */
 #define COPY_BYTE(N)			\
-	LOADBU(t0, N(src), .Ll_exc_copy);	\
+	LOADBU(t0, N(src), .Ll_exc_copy\@);	\
 	SUB	len, len, 1;		\
-	STOREB(t0, N(dst), .Ls_exc);	\
+	STOREB(t0, N(dst), .Ls_exc\@);	\
 	SLLV	t0, t0, t3;		\
 	addu	t3, SHIFT_INC;		\
-	beqz	len, .Lcopy_bytes_done; \
+	beqz	len, .Lcopy_bytes_done\@; \
 	 or	t2, t0
 
 	COPY_BYTE(0)
@@ -678,14 +688,14 @@ FEXPORT(csum_partial_copy_nocheck)
 	COPY_BYTE(4)
 	COPY_BYTE(5)
 #endif
-	LOADBU(t0, NBYTES-2(src), .Ll_exc_copy)
+	LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
 	SUB	len, len, 1
-	STOREB(t0, NBYTES-2(dst), .Ls_exc)
+	STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
 	SLLV	t0, t0, t3
 	or	t2, t0
-.Lcopy_bytes_done:
+.Lcopy_bytes_done\@:
 	ADDC(sum, t2)
-.Ldone:
+.Ldone\@:
 	/* fold checksum */
 #ifdef USE_DOUBLE
 	dsll32	v1, sum, 0
@@ -714,7 +724,7 @@ FEXPORT(csum_partial_copy_nocheck)
 	jr	ra
 	.set noreorder
 
-.Ll_exc_copy:
+.Ll_exc_copy\@:
 	/*
 	 * Copy bytes from src until faulting load address (or until a
 	 * lb faults)
@@ -729,7 +739,7 @@ FEXPORT(csum_partial_copy_nocheck)
 	 li	t2, SHIFT_START
 	LOADK	t0, THREAD_BUADDR(t0)
 1:
-	LOADBU(t1, 0(src), .Ll_exc)
+	LOADBU(t1, 0(src), .Ll_exc\@)
 	ADD	src, src, 1
 	sb	t1, 0(dst)	# can't fault -- we're copy_from_user
 	SLLV	t1, t1, t2
@@ -739,7 +749,7 @@ FEXPORT(csum_partial_copy_nocheck)
 	ADD	dst, dst, 1
 	bne	src, t0, 1b
 	.set	noreorder
-.Ll_exc:
+.Ll_exc\@:
 	LOADK	t0, TI_TASK($28)
 	 nop
 	LOADK	t0, THREAD_BUADDR(t0)	# t0 is just past last good address
@@ -758,7 +768,7 @@ FEXPORT(csum_partial_copy_nocheck)
 	 */
 	.set	reorder				/* DADDI_WAR */
 	SUB	src, len, 1
-	beqz	len, .Ldone
+	beqz	len, .Ldone\@
 	.set	noreorder
 1:	sb	zero, 0(dst)
 	ADD	dst, dst, 1
@@ -773,13 +783,19 @@ FEXPORT(csum_partial_copy_nocheck)
 	 SUB	src, src, v1
 #endif
 	li	v1, -EFAULT
-	b	.Ldone
+	b	.Ldone\@
 	 sw	v1, (errptr)
 
-.Ls_exc:
+.Ls_exc\@:
 	li	v0, -1 /* invalid checksum */
 	li	v1, -EFAULT
 	jr	ra
 	 sw	v1, (errptr)
 	.set	pop
-	END(__csum_partial_copy_kernel)
+	.endm
+
+LEAF(__csum_partial_copy_kernel)
+FEXPORT(__csum_partial_copy_to_user)
+FEXPORT(__csum_partial_copy_from_user)
+__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
+END(__csum_partial_copy_kernel)