[PATCH] m68k: clean up uaccess.h

This uninlines a few large functions in uaccess.h and cleans up the rest.
It includes a (hopefully temporary) workaround for the broken typeof of
gcc-4.1.

Signed-off-by: Roman Zippel <zippel@linux-m68k.org>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 271 insertions, 813 deletions

diff --git a/include/asm-m68k/uaccess.h b/include/asm-m68k/uaccess.h
index 2ffd87b..b761ef2 100644
--- a/include/asm-m68k/uaccess.h
+++ b/include/asm-m68k/uaccess.h
@@ -4,8 +4,9 @@
 /*
  * User space memory access functions
  */
+#include <linux/compiler.h>
 #include <linux/errno.h>
-#include <linux/sched.h>
+#include <linux/types.h>
 #include <asm/segment.h>
 
 #define VERIFY_READ	0
@@ -32,858 +33,315 @@ struct exception_table_entry
 	unsigned long insn, fixup;
 };
 
+extern int __put_user_bad(void);
+extern int __get_user_bad(void);
+
+#define __put_user_asm(res, x, ptr, bwl, reg, err)	\
+asm volatile ("\n"					\
+	"1:	moves."#bwl"	%2,%1\n"		\
+	"2:\n"						\
+	"	.section .fixup,\"ax\"\n"		\
+	"	.even\n"				\
+	"10:	moveq.l	%3,%0\n"			\
+	"	jra 2b\n"				\
+	"	.previous\n"				\
+	"\n"						\
+	"	.section __ex_table,\"a\"\n"		\
+	"	.align	4\n"				\
+	"	.long	1b,10b\n"			\
+	"	.long	2b,10b\n"			\
+	"	.previous"				\
+	: "+d" (res), "=m" (*(ptr))			\
+	: #reg (x), "i" (err))
 
 /*
  * These are the main single-value transfer routines.  They automatically
  * use the right size if we just have the right pointer type.
  */
 
-#define put_user(x, ptr)				\
-({							\
-    int __pu_err;					\
-    typeof(*(ptr)) __pu_val = (x);			\
-    __chk_user_ptr(ptr);				\
-    switch (sizeof (*(ptr))) {				\
-    case 1:						\
-	__put_user_asm(__pu_err, __pu_val, ptr, b);	\
-	break;						\
-    case 2:						\
-	__put_user_asm(__pu_err, __pu_val, ptr, w);	\
-	break;						\
-    case 4:						\
-	__put_user_asm(__pu_err, __pu_val, ptr, l);	\
-	break;						\
-    case 8:                                             \
-       __pu_err = __constant_copy_to_user(ptr, &__pu_val, 8);        \
-       break;                                           \
-    default:						\
-	__pu_err = __put_user_bad();			\
-	break;						\
-    }							\
-    __pu_err;						\
+#define __put_user(x, ptr)						\
+({									\
+	typeof(*(ptr)) __pu_val = (x);					\
+	int __pu_err = 0;						\
+	__chk_user_ptr(ptr);						\
+	switch (sizeof (*(ptr))) {					\
+	case 1:								\
+		__put_user_asm(__pu_err, __pu_val, ptr, b, d, -EFAULT);	\
+		break;							\
+	case 2:								\
+		__put_user_asm(__pu_err, __pu_val, ptr, w, d, -EFAULT);	\
+		break;							\
+	case 4:								\
+		__put_user_asm(__pu_err, __pu_val, ptr, l, r, -EFAULT);	\
+		break;							\
+	case 8:								\
+ 	    {								\
+ 		const void *__pu_ptr = (ptr);				\
+		asm volatile ("\n"					\
+			"1:	moves.l	%2,(%1)+\n"			\
+			"2:	moves.l	%R2,(%1)\n"			\
+			"3:\n"						\
+			"	.section .fixup,\"ax\"\n"		\
+			"	.even\n"				\
+			"10:	movel %3,%0\n"				\
+			"	jra 3b\n"				\
+			"	.previous\n"				\
+			"\n"						\
+			"	.section __ex_table,\"a\"\n"		\
+			"	.align 4\n"				\
+			"	.long 1b,10b\n"				\
+			"	.long 2b,10b\n"				\
+			"	.long 3b,10b\n"				\
+			"	.previous"				\
+			: "+d" (__pu_err), "+a" (__pu_ptr)		\
+			: "r" (__pu_val), "i" (-EFAULT)			\
+			: "memory");					\
+		break;							\
+	    }								\
+	default:							\
+		__pu_err = __put_user_bad();				\
+		break;							\
+	}								\
+	__pu_err;							\
 })
-#define __put_user(x, ptr) put_user(x, ptr)
+#define put_user(x, ptr)	__put_user(x, ptr)
 
-extern int __put_user_bad(void);
 
-/*
- * Tell gcc we read from memory instead of writing: this is because
- * we do not write to any memory gcc knows about, so there are no
- * aliasing issues.
- */
-#define __put_user_asm(err,x,ptr,bwl)			\
-__asm__ __volatile__					\
-    ("21:moves" #bwl " %2,%1\n"				\
-     "1:\n"						\
-     ".section .fixup,\"ax\"\n"				\
-     "   .even\n"					\
-     "2: movel %3,%0\n"					\
-     "   jra 1b\n"					\
-     ".previous\n"					\
-     ".section __ex_table,\"a\"\n"			\
-     "   .align 4\n"					\
-     "   .long 21b,2b\n"				\
-     "   .long 1b,2b\n"					\
-     ".previous"					\
-     : "=d"(err)					\
-     : "m"(*(ptr)), "r"(x), "i"(-EFAULT), "0"(0))
-
-#define get_user(x, ptr)					\
-({								\
-    int __gu_err;						\
-    typeof(*(ptr)) __gu_val;					\
-    __chk_user_ptr(ptr);					\
-    switch (sizeof(*(ptr))) {					\
-    case 1:							\
-	__get_user_asm(__gu_err, __gu_val, ptr, b, "=d");	\
-	break;							\
-    case 2:							\
-	__get_user_asm(__gu_err, __gu_val, ptr, w, "=r");	\
-	break;							\
-    case 4:							\
-	__get_user_asm(__gu_err, __gu_val, ptr, l, "=r");	\
-	break;							\
-    case 8:                                                     \
-        __gu_err = __constant_copy_from_user(&__gu_val, ptr, 8);  \
-        break;                                                  \
-    default:							\
-	__gu_val = (typeof(*(ptr)))0;				\
-	__gu_err = __get_user_bad();				\
-	break;							\
-    }								\
-    (x) = __gu_val;						\
-    __gu_err;							\
+#define __get_user_asm(res, x, ptr, type, bwl, reg, err) ({	\
+	type __gu_val;						\
+	asm volatile ("\n"					\
+		"1:	moves."#bwl"	%2,%1\n"		\
+		"2:\n"						\
+		"	.section .fixup,\"ax\"\n"		\
+		"	.even\n"				\
+		"10:	move.l	%3,%0\n"			\
+		"	sub."#bwl"	%1,%1\n"		\
+		"	jra	2b\n"				\
+		"	.previous\n"				\
+		"\n"						\
+		"	.section __ex_table,\"a\"\n"		\
+		"	.align	4\n"				\
+		"	.long	1b,10b\n"			\
+		"	.previous"				\
+		: "+d" (res), "=&" #reg (__gu_val)		\
+		: "m" (*(ptr)), "i" (err));			\
+	(x) = (typeof(*(ptr)))(long)__gu_val;			\
 })
-#define __get_user(x, ptr) get_user(x, ptr)
 
-extern int __get_user_bad(void);
+#define __get_user(x, ptr)						\
+({									\
+	int __gu_err = 0;						\
+	__chk_user_ptr(ptr);						\
+	switch (sizeof(*(ptr))) {					\
+	case 1:								\
+		__get_user_asm(__gu_err, x, ptr, u8, b, d, -EFAULT);	\
+		break;							\
+	case 2:								\
+		__get_user_asm(__gu_err, x, ptr, u16, w, d, -EFAULT);	\
+		break;							\
+	case 4:								\
+		__get_user_asm(__gu_err, x, ptr, u32, l, r, -EFAULT);	\
+		break;							\
+/*	case 8:	disabled because gcc-4.1 has a broken typeof		\
+ 	    {								\
+ 		const void *__gu_ptr = (ptr);				\
+ 		u64 __gu_val;						\
+		asm volatile ("\n"					\
+			"1:	moves.l	(%2)+,%1\n"			\
+			"2:	moves.l	(%2),%R1\n"			\
+			"3:\n"						\
+			"	.section .fixup,\"ax\"\n"		\
+			"	.even\n"				\
+			"10:	move.l	%3,%0\n"			\
+			"	sub.l	%1,%1\n"			\
+			"	sub.l	%R1,%R1\n"			\
+			"	jra	3b\n"				\
+			"	.previous\n"				\
+			"\n"						\
+			"	.section __ex_table,\"a\"\n"		\
+			"	.align	4\n"				\
+			"	.long	1b,10b\n"			\
+			"	.long	2b,10b\n"			\
+			"	.previous"				\
+			: "+d" (__gu_err), "=&r" (__gu_val),		\
+			  "+a" (__gu_ptr)				\
+			: "i" (-EFAULT)					\
+			: "memory");					\
+		(x) = (typeof(*(ptr)))__gu_val;				\
+		break;							\
+	    }	*/							\
+	default:							\
+		__gu_err = __get_user_bad();				\
+		break;							\
+	}								\
+	__gu_err;							\
+})
+#define get_user(x, ptr) __get_user(x, ptr)
 
-#define __get_user_asm(err,x,ptr,bwl,reg)	\
-__asm__ __volatile__				\
-    ("1: moves" #bwl " %2,%1\n"			\
-     "2:\n"					\
-     ".section .fixup,\"ax\"\n"			\
-     "   .even\n"				\
-     "3: movel %3,%0\n"				\
-     "   sub" #bwl " %1,%1\n"			\
-     "   jra 2b\n"				\
-     ".previous\n"				\
-     ".section __ex_table,\"a\"\n"		\
-     "   .align 4\n"				\
-     "   .long 1b,3b\n"				\
-     ".previous"				\
-     : "=d"(err), reg(x)			\
-     : "m"(*(ptr)), "i" (-EFAULT), "0"(0))
+unsigned long __generic_copy_from_user(void *to, const void __user *from, unsigned long n);
+unsigned long __generic_copy_to_user(void __user *to, const void *from, unsigned long n);
 
-static inline unsigned long
-__generic_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-    unsigned long tmp;
-    __asm__ __volatile__
-	("   tstl %2\n"
-	 "   jeq 2f\n"
-	 "1: movesl (%1)+,%3\n"
-	 "   movel %3,(%0)+\n"
-	 "   subql #1,%2\n"
-	 "   jne 1b\n"
-	 "2: movel %4,%2\n"
-	 "   bclr #1,%2\n"
-	 "   jeq 4f\n"
-	 "3: movesw (%1)+,%3\n"
-	 "   movew %3,(%0)+\n"
-	 "4: bclr #0,%2\n"
-	 "   jeq 6f\n"
-	 "5: movesb (%1)+,%3\n"
-	 "   moveb %3,(%0)+\n"
-	 "6:\n"
-	 ".section .fixup,\"ax\"\n"
-	 "   .even\n"
-	 "7: movel %2,%%d0\n"
-	 "71:clrl (%0)+\n"
-	 "   subql #1,%%d0\n"
-	 "   jne 71b\n"
-	 "   lsll #2,%2\n"
-	 "   addl %4,%2\n"
-	 "   btst #1,%4\n"
-	 "   jne 81f\n"
-	 "   btst #0,%4\n"
-	 "   jne 91f\n"
-	 "   jra 6b\n"
-	 "8: addql #2,%2\n"
-	 "81:clrw (%0)+\n"
-	 "   btst #0,%4\n"
-	 "   jne 91f\n"
-	 "   jra 6b\n"
-	 "9: addql #1,%2\n"
-	 "91:clrb (%0)+\n"
-	 "   jra 6b\n"
-         ".previous\n"
-	 ".section __ex_table,\"a\"\n"
-	 "   .align 4\n"
-	 "   .long 1b,7b\n"
-	 "   .long 3b,8b\n"
-	 "   .long 5b,9b\n"
-	 ".previous"
-	 : "=a"(to), "=a"(from), "=d"(n), "=&d"(tmp)
-	 : "d"(n & 3), "0"(to), "1"(from), "2"(n/4)
-	 : "d0", "memory");
-    return n;
-}
-
-static inline unsigned long
-__generic_copy_to_user(void __user *to, const void *from, unsigned long n)
+static __always_inline unsigned long
+__constant_copy_from_user(void *to, const void __user *from, unsigned long n)
 {
-    unsigned long tmp;
-    __asm__ __volatile__
-	("   tstl %2\n"
-	 "   jeq 3f\n"
-	 "1: movel (%1)+,%3\n"
-	 "22:movesl %3,(%0)+\n"
-	 "2: subql #1,%2\n"
-	 "   jne 1b\n"
-	 "3: movel %4,%2\n"
-	 "   bclr #1,%2\n"
-	 "   jeq 4f\n"
-	 "   movew (%1)+,%3\n"
-	 "24:movesw %3,(%0)+\n"
-	 "4: bclr #0,%2\n"
-	 "   jeq 5f\n"
-	 "   moveb (%1)+,%3\n"
-	 "25:movesb %3,(%0)+\n"
-	 "5:\n"
-	 ".section .fixup,\"ax\"\n"
-	 "   .even\n"
-	 "60:addql #1,%2\n"
-	 "6: lsll #2,%2\n"
-	 "   addl %4,%2\n"
-	 "   jra 5b\n"
-	 "7: addql #2,%2\n"
-	 "   jra 5b\n"
-	 "8: addql #1,%2\n"
-	 "   jra 5b\n"
-	 ".previous\n"
-	 ".section __ex_table,\"a\"\n"
-	 "   .align 4\n"
-	 "   .long 1b,60b\n"
-	 "   .long 22b,6b\n"
-	 "   .long 2b,6b\n"
-	 "   .long 24b,7b\n"
-	 "   .long 3b,60b\n"
-	 "   .long 4b,7b\n"
-	 "   .long 25b,8b\n"
-	 "   .long 5b,8b\n"
-	 ".previous"
-	 : "=a"(to), "=a"(from), "=d"(n), "=&d"(tmp)
-	 : "r"(n & 3), "0"(to), "1"(from), "2"(n / 4)
-	 : "memory");
-    return n;
-}
+	unsigned long res = 0, tmp;
 
-#define __copy_from_user_big(to, from, n, fixup, copy)	\
-    __asm__ __volatile__				\
-	("10: movesl (%1)+,%%d0\n"			\
-	 "    movel %%d0,(%0)+\n"			\
-	 "    subql #1,%2\n"				\
-	 "    jne 10b\n"				\
-	 ".section .fixup,\"ax\"\n"			\
-	 "    .even\n"					\
-	 "11: movel %2,%%d0\n"				\
-	 "13: clrl (%0)+\n"				\
-	 "    subql #1,%%d0\n"				\
-	 "    jne 13b\n"				\
-	 "    lsll #2,%2\n"				\
-	 fixup "\n"					\
-	 "    jra 12f\n"				\
-	 ".previous\n"					\
-	 ".section __ex_table,\"a\"\n"			\
-	 "    .align 4\n"				\
-	 "    .long 10b,11b\n"				\
-	 ".previous\n"					\
-	 copy "\n"					\
-	 "12:"						\
-	 : "=a"(to), "=a"(from), "=d"(n)		\
-	 : "0"(to), "1"(from), "2"(n/4)			\
-	 : "d0", "memory")
+	/* limit the inlined version to 3 moves */
+	if (n == 11 || n > 12)
+		return __generic_copy_from_user(to, from, n);
 
-static inline unsigned long
-__constant_copy_from_user(void *to, const void __user *from, unsigned long n)
-{
-    switch (n) {
-    case 0:
-	break;
-    case 1:
-	__asm__ __volatile__
-	    ("1: movesb (%1)+,%%d0\n"
-	     "   moveb %%d0,(%0)+\n"
-	     "2:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "3: addql #1,%2\n"
-	     "   clrb (%0)+\n"
-	     "   jra 2b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,3b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 2:
-	__asm__ __volatile__
-	    ("1: movesw (%1)+,%%d0\n"
-	     "   movew %%d0,(%0)+\n"
-	     "2:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "3: addql #2,%2\n"
-	     "   clrw (%0)+\n"
-	     "   jra 2b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,3b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 3:
-	__asm__ __volatile__
-	    ("1: movesw (%1)+,%%d0\n"
-	     "   movew %%d0,(%0)+\n"
-	     "2: movesb (%1)+,%%d0\n"
-	     "   moveb %%d0,(%0)+\n"
-	     "3:"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "4: addql #2,%2\n"
-	     "   clrw (%0)+\n"
-	     "5: addql #1,%2\n"
-	     "   clrb (%0)+\n"
-	     "   jra 3b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,4b\n"
-	     "   .long 2b,5b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 4:
-	__asm__ __volatile__
-	    ("1: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "2:"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "3: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "   jra 2b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,3b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 8:
-	__asm__ __volatile__
-	    ("1: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "2: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "3:"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "4: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "5: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "   jra 3b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,4b\n"
-	     "   .long 2b,5b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 12:
-	__asm__ __volatile__
-	    ("1: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "2: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "3: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "4:"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "5: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "6: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "7: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "   jra 4b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,5b\n"
-	     "   .long 2b,6b\n"
-	     "   .long 3b,7b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 16:
-	__asm__ __volatile__
-	    ("1: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "2: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "3: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "4: movesl (%1)+,%%d0\n"
-	     "   movel %%d0,(%0)+\n"
-	     "5:"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "6: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "7: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "8: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "9: addql #4,%2\n"
-	     "   clrl (%0)+\n"
-	     "   jra 5b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 1b,6b\n"
-	     "   .long 2b,7b\n"
-	     "   .long 3b,8b\n"
-	     "   .long 4b,9b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    default:
-	switch (n & 3) {
-	case 0:
-	    __copy_from_user_big(to, from, n, "", "");
-	    break;
+	switch (n) {
 	case 1:
-	    __copy_from_user_big(to, from, n,
-				 /* fixup */
-				 "1: addql #1,%2\n"
-				 "   clrb (%0)+",
-				 /* copy */
-				 "2: movesb (%1)+,%%d0\n"
-				 "   moveb %%d0,(%0)+\n"
-				 ".section __ex_table,\"a\"\n"
-				 "   .long 2b,1b\n"
-				 ".previous");
-	    break;
+		__get_user_asm(res, *(u8 *)to, (u8 *)from, u8, b, d, 1);
+		return res;
 	case 2:
-	    __copy_from_user_big(to, from, n,
-				 /* fixup */
-				 "1: addql #2,%2\n"
-				 "   clrw (%0)+",
-				 /* copy */
-				 "2: movesw (%1)+,%%d0\n"
-				 "   movew %%d0,(%0)+\n"
-				 ".section __ex_table,\"a\"\n"
-				 "   .long 2b,1b\n"
-				 ".previous");
-	    break;
-	case 3:
-	    __copy_from_user_big(to, from, n,
-				 /* fixup */
-				 "1: addql #2,%2\n"
-				 "   clrw (%0)+\n"
-				 "2: addql #1,%2\n"
-				 "   clrb (%0)+",
-				 /* copy */
-				 "3: movesw (%1)+,%%d0\n"
-				 "   movew %%d0,(%0)+\n"
-				 "4: movesb (%1)+,%%d0\n"
-				 "   moveb %%d0,(%0)+\n"
-				 ".section __ex_table,\"a\"\n"
-				 "   .long 3b,1b\n"
-				 "   .long 4b,2b\n"
-				 ".previous");
-	    break;
+		__get_user_asm(res, *(u16 *)to, (u16 *)from, u16, w, d, 2);
+		return res;
+	case 4:
+		__get_user_asm(res, *(u32 *)to, (u32 *)from, u32, l, r, 4);
+		return res;
 	}
-	break;
-    }
-    return n;
-}
 
-#define __copy_to_user_big(to, from, n, fixup, copy)	\
-    __asm__ __volatile__				\
-	("10: movel (%1)+,%%d0\n"			\
-	 "31: movesl %%d0,(%0)+\n"			\
-	 "11: subql #1,%2\n"				\
-	 "    jne 10b\n"				\
-	 "41:\n"					\
-	 ".section .fixup,\"ax\"\n"			\
-	 "   .even\n"					\
-	 "22: addql #1,%2\n"				\
-	 "12: lsll #2,%2\n"				\
-	 fixup "\n"					\
-	 "    jra 13f\n"				\
-	 ".previous\n"					\
-	 ".section __ex_table,\"a\"\n"			\
-	 "    .align 4\n"				\
-	 "    .long 10b,22b\n"				\
-	 "    .long 31b,12b\n"				\
-	 "    .long 11b,12b\n"				\
-	 "    .long 41b,22b\n"				\
-	 ".previous\n"					\
-	 copy "\n"					\
-	 "13:"						\
-	 : "=a"(to), "=a"(from), "=d"(n)		\
-	 : "0"(to), "1"(from), "2"(n/4)			\
-	 : "d0", "memory")
+	asm volatile ("\n"
+		"	.ifndef	.Lfrom_user\n"
+		"	.set	.Lfrom_user,1\n"
+		"	.macro	copy_from_user to,from,tmp\n"
+		"	.if	.Lcnt >= 4\n"
+		"1:	moves.l	(\\from)+,\\tmp\n"
+		"	move.l	\\tmp,(\\to)+\n"
+		"	.set	.Lcnt,.Lcnt-4\n"
+		"	.elseif	.Lcnt & 2\n"
+		"1:	moves.w	(\\from)+,\\tmp\n"
+		"	move.w	\\tmp,(\\to)+\n"
+		"	.set	.Lcnt,.Lcnt-2\n"
+		"	.elseif	.Lcnt & 1\n"
+		"1:	moves.b	(\\from)+,\\tmp\n"
+		"	move.b	\\tmp,(\\to)+\n"
+		"	.set	.Lcnt,.Lcnt-1\n"
+		"	.else\n"
+		"	.exitm\n"
+		"	.endif\n"
+		"\n"
+		"	.section __ex_table,\"a\"\n"
+		"	.align	4\n"
+		"	.long	1b,3f\n"
+		"	.previous\n"
+		"	.endm\n"
+		"	.endif\n"
+		"\n"
+		"	.set	.Lcnt,%c4\n"
+		"	copy_from_user %1,%2,%3\n"
+		"	copy_from_user %1,%2,%3\n"
+		"	copy_from_user %1,%2,%3\n"
+		"2:\n"
+		"	.section .fixup,\"ax\"\n"
+		"	.even\n"
+		"3:	moveq.l	%4,%0\n"
+		"	move.l	%5,%1\n"
+		"	.rept	%c4 / 4\n"
+		"	clr.l	(%1)+\n"
+		"	.endr\n"
+		"	.if	%c4 & 2\n"
+		"	clr.w	(%1)+\n"
+		"	.endif\n"
+		"	.if	%c4 & 1\n"
+		"	clr.b	(%1)+\n"
+		"	.endif\n"
+		"	jra	2b\n"
+		"	.previous\n"
+		: "+r" (res), "+a" (to), "+a" (from), "=&d" (tmp)
+		: "i" (n), "g" (to)
+		: "memory");
 
-#define __copy_to_user_inatomic __copy_to_user
-#define __copy_from_user_inatomic __copy_from_user
+	return res;
+}
 
-static inline unsigned long
+static __always_inline unsigned long
 __constant_copy_to_user(void __user *to, const void *from, unsigned long n)
 {
-    switch (n) {
-    case 0:
-	break;
-    case 1:
-	__asm__ __volatile__
-	    ("   moveb (%1)+,%%d0\n"
-	     "21:movesb %%d0,(%0)+\n"
-	     "1:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "2: addql #1,%2\n"
-	     "   jra 1b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n  "
-	     "   .long 21b,2b\n"
-	     "   .long 1b,2b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 2:
-	__asm__ __volatile__
-	    ("   movew (%1)+,%%d0\n"
-	     "21:movesw %%d0,(%0)+\n"
-	     "1:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "2: addql #2,%2\n"
-	     "   jra 1b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 21b,2b\n"
-	     "   .long 1b,2b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 3:
-	__asm__ __volatile__
-	    ("   movew (%1)+,%%d0\n"
-	     "21:movesw %%d0,(%0)+\n"
-	     "1: moveb (%1)+,%%d0\n"
-	     "22:movesb %%d0,(%0)+\n"
-	     "2:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "3: addql #2,%2\n"
-	     "4: addql #1,%2\n"
-	     "   jra 2b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 21b,3b\n"
-	     "   .long 1b,3b\n"
-	     "   .long 22b,4b\n"
-	     "   .long 2b,4b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 4:
-	__asm__ __volatile__
-	    ("   movel (%1)+,%%d0\n"
-	     "21:movesl %%d0,(%0)+\n"
-	     "1:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "2: addql #4,%2\n"
-	     "   jra 1b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 21b,2b\n"
-	     "   .long 1b,2b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 8:
-	__asm__ __volatile__
-	    ("   movel (%1)+,%%d0\n"
-	     "21:movesl %%d0,(%0)+\n"
-	     "1: movel (%1)+,%%d0\n"
-	     "22:movesl %%d0,(%0)+\n"
-	     "2:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "3: addql #4,%2\n"
-	     "4: addql #4,%2\n"
-	     "   jra 2b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 21b,3b\n"
-	     "   .long 1b,3b\n"
-	     "   .long 22b,4b\n"
-	     "   .long 2b,4b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 12:
-	__asm__ __volatile__
-	    ("   movel (%1)+,%%d0\n"
-	     "21:movesl %%d0,(%0)+\n"
-	     "1: movel (%1)+,%%d0\n"
-	     "22:movesl %%d0,(%0)+\n"
-	     "2: movel (%1)+,%%d0\n"
-	     "23:movesl %%d0,(%0)+\n"
-	     "3:\n"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "4: addql #4,%2\n"
-	     "5: addql #4,%2\n"
-	     "6: addql #4,%2\n"
-	     "   jra 3b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 21b,4b\n"
-	     "   .long 1b,4b\n"
-	     "   .long 22b,5b\n"
-	     "   .long 2b,5b\n"
-	     "   .long 23b,6b\n"
-	     "   .long 3b,6b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    case 16:
-	__asm__ __volatile__
-	    ("   movel (%1)+,%%d0\n"
-	     "21:movesl %%d0,(%0)+\n"
-	     "1: movel (%1)+,%%d0\n"
-	     "22:movesl %%d0,(%0)+\n"
-	     "2: movel (%1)+,%%d0\n"
-	     "23:movesl %%d0,(%0)+\n"
-	     "3: movel (%1)+,%%d0\n"
-	     "24:movesl %%d0,(%0)+\n"
-	     "4:"
-	     ".section .fixup,\"ax\"\n"
-	     "   .even\n"
-	     "5: addql #4,%2\n"
-	     "6: addql #4,%2\n"
-	     "7: addql #4,%2\n"
-	     "8: addql #4,%2\n"
-	     "   jra 4b\n"
-	     ".previous\n"
-	     ".section __ex_table,\"a\"\n"
-	     "   .align 4\n"
-	     "   .long 21b,5b\n"
-	     "   .long 1b,5b\n"
-	     "   .long 22b,6b\n"
-	     "   .long 2b,6b\n"
-	     "   .long 23b,7b\n"
-	     "   .long 3b,7b\n"
-	     "   .long 24b,8b\n"
-	     "   .long 4b,8b\n"
-	     ".previous"
-	     : "=a"(to), "=a"(from), "=d"(n)
-	     : "0"(to), "1"(from), "2"(0)
-	     : "d0", "memory");
-	break;
-    default:
-	switch (n & 3) {
-	case 0:
-	    __copy_to_user_big(to, from, n, "", "");
-	    break;
+	unsigned long res = 0, tmp;
+
+	/* limit the inlined version to 3 moves */
+	if (n == 11 || n > 12)
+		return __generic_copy_to_user(to, from, n);
+
+	switch (n) {
 	case 1:
-	    __copy_to_user_big(to, from, n,
-			       /* fixup */
-			       "1: addql #1,%2",
-			       /* copy */
-			       "   moveb (%1)+,%%d0\n"
-			       "22:movesb %%d0,(%0)+\n"
-			       "2:"
-			       ".section __ex_table,\"a\"\n"
-			       "   .long 22b,1b\n"
-			       "   .long 2b,1b\n"
-			       ".previous");
-	    break;
+		__put_user_asm(res, *(u8 *)from, (u8 *)to, b, d, 1);
+		return res;
 	case 2:
-	    __copy_to_user_big(to, from, n,
-			       /* fixup */
-			       "1: addql #2,%2",
-			       /* copy */
-			       "   movew (%1)+,%%d0\n"
-			       "22:movesw %%d0,(%0)+\n"
-			       "2:"
-			       ".section __ex_table,\"a\"\n"
-			       "   .long 22b,1b\n"
-			       "   .long 2b,1b\n"
-			       ".previous");
-	    break;
-	case 3:
-	    __copy_to_user_big(to, from, n,
-			       /* fixup */
-			       "1: addql #2,%2\n"
-			       "2: addql #1,%2",
-			       /* copy */
-			       "   movew (%1)+,%%d0\n"
-			       "23:movesw %%d0,(%0)+\n"
-			       "3: moveb (%1)+,%%d0\n"
-			       "24:movesb %%d0,(%0)+\n"
-			       "4:"
-			       ".section __ex_table,\"a\"\n"
-			       "   .long 23b,1b\n"
-			       "   .long 3b,1b\n"
-			       "   .long 24b,2b\n"
-			       "   .long 4b,2b\n"
-			       ".previous");
-	    break;
+		__put_user_asm(res, *(u16 *)from, (u16 *)to, w, d, 2);
+		return res;
+	case 4:
+		__put_user_asm(res, *(u32 *)from, (u32 *)to, l, r, 4);
+		return res;
 	}
-	break;
-    }
-    return n;
+
+	asm volatile ("\n"
+		"	.ifndef	.Lto_user\n"
+		"	.set	.Lto_user,1\n"
+		"	.macro	copy_to_user to,from,tmp\n"
+		"	.if	.Lcnt >= 4\n"
+		"	move.l	(\\from)+,\\tmp\n"
+		"11:	moves.l	\\tmp,(\\to)+\n"
+		"12:	.set	.Lcnt,.Lcnt-4\n"
+		"	.elseif	.Lcnt & 2\n"
+		"	move.w	(\\from)+,\\tmp\n"
+		"11:	moves.w	\\tmp,(\\to)+\n"
+		"12:	.set	.Lcnt,.Lcnt-2\n"
+		"	.elseif	.Lcnt & 1\n"
+		"	move.b	(\\from)+,\\tmp\n"
+		"11:	moves.b	\\tmp,(\\to)+\n"
+		"12:	.set	.Lcnt,.Lcnt-1\n"
+		"	.else\n"
+		"	.exitm\n"
+		"	.endif\n"
+		"\n"
+		"	.section __ex_table,\"a\"\n"
+		"	.align	4\n"
+		"	.long	11b,3f\n"
+		"	.long	12b,3f\n"
+		"	.previous\n"
+		"	.endm\n"
+		"	.endif\n"
+		"\n"
+		"	.set	.Lcnt,%c4\n"
+		"	copy_to_user %1,%2,%3\n"
+		"	copy_to_user %1,%2,%3\n"
+		"	copy_to_user %1,%2,%3\n"
+		"2:\n"
+		"	.section .fixup,\"ax\"\n"
+		"	.even\n"
+		"3:	moveq.l	%4,%0\n"
+		"	jra	2b\n"
+		"	.previous\n"
+		: "+r" (res), "+a" (to), "+a" (from), "=&d" (tmp)
+		: "i" (n)
+		: "memory");
+
+	return res;
 }
 
-#define copy_from_user(to, from, n)		\
+#define __copy_from_user(to, from, n)		\
 (__builtin_constant_p(n) ?			\
  __constant_copy_from_user(to, from, n) :	\
  __generic_copy_from_user(to, from, n))
 
-#define copy_to_user(to, from, n)		\
+#define __copy_to_user(to, from, n)		\
 (__builtin_constant_p(n) ?			\
  __constant_copy_to_user(to, from, n) :		\
  __generic_copy_to_user(to, from, n))
 
-#define __copy_from_user(to, from, n) copy_from_user(to, from, n)
-#define __copy_to_user(to, from, n) copy_to_user(to, from, n)
+#define __copy_to_user_inatomic		__copy_to_user
+#define __copy_from_user_inatomic	__copy_from_user
 
-/*
- * Copy a null terminated string from userspace.
- */
+#define copy_from_user(to, from, n)	__copy_from_user(to, from, n)
+#define copy_to_user(to, from, n)	__copy_to_user(to, from, n)
 
-static inline long
-strncpy_from_user(char *dst, const char __user *src, long count)
-{
-    long res;
-    if (count == 0) return count;
-    __asm__ __volatile__
-	("1: movesb (%2)+,%%d0\n"
-	 "12:moveb %%d0,(%1)+\n"
-	 "   jeq 2f\n"
-	 "   subql #1,%3\n"
-	 "   jne 1b\n"
-	 "2: subl %3,%0\n"
-	 "3:\n"
-	 ".section .fixup,\"ax\"\n"
-	 "   .even\n"
-	 "4: movel %4,%0\n"
-	 "   jra 3b\n"
-	 ".previous\n"
-	 ".section __ex_table,\"a\"\n"
-	 "   .align 4\n"
-	 "   .long 1b,4b\n"
-	 "   .long 12b,4b\n"
-	 ".previous"
-	 : "=d"(res), "=a"(dst), "=a"(src), "=d"(count)
-	 : "i"(-EFAULT), "0"(count), "1"(dst), "2"(src), "3"(count)
-	 : "d0", "memory");
-    return res;
-}
-
-/*
- * Return the size of a string (including the ending 0)
- *
- * Return 0 on exception, a value greater than N if too long
- */
-static inline long strnlen_user(const char __user *src, long n)
-{
-	long res;
-
-	res = -(unsigned long)src;
-	__asm__ __volatile__
-		("1:\n"
-		 "   tstl %2\n"
-		 "   jeq 3f\n"
-		 "2: movesb (%1)+,%%d0\n"
-		 "22:\n"
-		 "   subql #1,%2\n"
-		 "   tstb %%d0\n"
-		 "   jne 1b\n"
-		 "   jra 4f\n"
-		 "3:\n"
-		 "   addql #1,%0\n"
-		 "4:\n"
-		 "   addl %1,%0\n"
-		 "5:\n"
-		 ".section .fixup,\"ax\"\n"
-		 "   .even\n"
-		 "6: moveq %3,%0\n"
-		 "   jra 5b\n"
-		 ".previous\n"
-		 ".section __ex_table,\"a\"\n"
-		 "   .align 4\n"
-		 "   .long 2b,6b\n"
-		 "   .long 22b,6b\n"
-		 ".previous"
-		 : "=d"(res), "=a"(src), "=d"(n)
-		 : "i"(0), "0"(res), "1"(src), "2"(n)
-		 : "d0");
-	return res;
-}
+long strncpy_from_user(char *dst, const char __user *src, long count);
+long strnlen_user(const char __user *src, long n);
+unsigned long clear_user(void __user *to, unsigned long n);
 
 #define strlen_user(str) strnlen_user(str, 32767)
 
-/*
- * Zero Userspace
- */
-
-static inline unsigned long
-clear_user(void __user *to, unsigned long n)
-{
-    __asm__ __volatile__
-	("   tstl %1\n"
-	 "   jeq 3f\n"
-	 "1: movesl %3,(%0)+\n"
-	 "2: subql #1,%1\n"
-	 "   jne 1b\n"
-	 "3: movel %2,%1\n"
-	 "   bclr #1,%1\n"
-	 "   jeq 4f\n"
-	 "24:movesw %3,(%0)+\n"
-	 "4: bclr #0,%1\n"
-	 "   jeq 5f\n"
-	 "25:movesb %3,(%0)+\n"
-	 "5:\n"
-	 ".section .fixup,\"ax\"\n"
-	 "   .even\n"
-	 "61:addql #1,%1\n"
-	 "6: lsll #2,%1\n"
-	 "   addl %2,%1\n"
-	 "   jra 5b\n"
-	 "7: addql #2,%1\n"
-	 "   jra 5b\n"
-	 "8: addql #1,%1\n"
-	 "   jra 5b\n"
-	 ".previous\n"
-	 ".section __ex_table,\"a\"\n"
-	 "   .align 4\n"
-	 "   .long 1b,61b\n"
-	 "   .long 2b,6b\n"
-	 "   .long 3b,61b\n"
-	 "   .long 24b,7b\n"
-	 "   .long 4b,7b\n"
-	 "   .long 25b,8b\n"
-	 "   .long 5b,8b\n"
-	 ".previous"
-	 : "=a"(to), "=d"(n)
-	 : "r"(n & 3), "r"(0), "0"(to), "1"(n/4));
-    return n;
-}
-
 #endif /* _M68K_UACCESS_H */