s390/uaccess: get rid of indirect function calls

There are only two uaccess variants on s390 left: the version that is used
if the mvcos instruction is available, and the page table walk variant.
So there is no need for expensive indirect function calls.

By default the mvcos variant will be called. If the mvcos instruction is not
available it will call the page table walk variant.

For minimal performance impact the "if (mvcos_is_available)" is implemented
with a jump label, which will be a six byte nop on machines with mvcos.

Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

4 files changed, 86 insertions, 45 deletions

diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index b068729..e3fffe1 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -2,8 +2,7 @@
 # Makefile for s390-specific library files..
 #
 
-lib-y += delay.o string.o uaccess_pt.o find.o
+lib-y += delay.o string.o uaccess_pt.o uaccess_mvcos.o find.o
 obj-$(CONFIG_32BIT) += div64.o qrnnd.o ucmpdi2.o mem32.o
 obj-$(CONFIG_64BIT) += mem64.o
-lib-$(CONFIG_64BIT) += uaccess_mvcos.o
 lib-$(CONFIG_SMP) += spinlock.o
diff --git a/arch/s390/lib/uaccess.h b/arch/s390/lib/uaccess.h
index b1a2217..e5b9c924 100644
--- a/arch/s390/lib/uaccess.h
+++ b/arch/s390/lib/uaccess.h
@@ -6,7 +6,11 @@
 #ifndef __ARCH_S390_LIB_UACCESS_H
 #define __ARCH_S390_LIB_UACCESS_H
 
-extern int futex_atomic_op_pt(int, u32 __user *, int, int *);
-extern int futex_atomic_cmpxchg_pt(u32 *, u32 __user *, u32, u32);
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n);
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n);
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n);
+size_t clear_user_pt(void __user *to, size_t n);
+size_t strnlen_user_pt(const char __user *src, size_t count);
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count);
 
 #endif /* __ARCH_S390_LIB_UACCESS_H */
diff --git a/arch/s390/lib/uaccess_mvcos.c b/arch/s390/lib/uaccess_mvcos.c
index 95123f5..66f35e1 100644
--- a/arch/s390/lib/uaccess_mvcos.c
+++ b/arch/s390/lib/uaccess_mvcos.c
@@ -6,7 +6,9 @@
  *		 Gerald Schaefer (gerald.schaefer@de.ibm.com)
  */
 
+#include <linux/jump_label.h>
 #include <linux/errno.h>
+#include <linux/init.h>
 #include <linux/mm.h>
 #include <asm/uaccess.h>
 #include <asm/futex.h>
@@ -26,7 +28,10 @@
 #define SLR	"slgr"
 #endif
 
-static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
+static struct static_key have_mvcos = STATIC_KEY_INIT_TRUE;
+
+static inline size_t copy_from_user_mvcos(void *x, const void __user *ptr,
+					  size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x81UL;
 	unsigned long tmp1, tmp2;
@@ -65,7 +70,16 @@ static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size)
 	return size;
 }
 
-static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
+size_t __copy_from_user(void *to, const void __user *from, size_t n)
+{
+	if (static_key_true(&have_mvcos))
+		return copy_from_user_mvcos(to, from, n);
+	return copy_from_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_from_user);
+
+static inline size_t copy_to_user_mvcos(void __user *ptr, const void *x,
+					size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x810000UL;
 	unsigned long tmp1, tmp2;
@@ -94,8 +108,16 @@ static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size)
 	return size;
 }
 
-static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
-				 size_t size)
+size_t __copy_to_user(void __user *to, const void *from, size_t n)
+{
+	if (static_key_true(&have_mvcos))
+		return copy_to_user_mvcos(to, from, n);
+	return copy_to_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_to_user);
+
+static inline size_t copy_in_user_mvcos(void __user *to, const void __user *from,
+					size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x810081UL;
 	unsigned long tmp1, tmp2;
@@ -117,7 +139,15 @@ static size_t copy_in_user_mvcos(void __user *to, const void __user *from,
 	return size;
 }
 
-static size_t clear_user_mvcos(void __user *to, size_t size)
+size_t __copy_in_user(void __user *to, const void __user *from, size_t n)
+{
+	if (static_key_true(&have_mvcos))
+		return copy_in_user_mvcos(to, from, n);
+	return copy_in_user_pt(to, from, n);
+}
+EXPORT_SYMBOL(__copy_in_user);
+
+static inline size_t clear_user_mvcos(void __user *to, size_t size)
 {
 	register unsigned long reg0 asm("0") = 0x810000UL;
 	unsigned long tmp1, tmp2;
@@ -145,7 +175,15 @@ static size_t clear_user_mvcos(void __user *to, size_t size)
 	return size;
 }
 
-static size_t strnlen_user_mvcos(const char __user *src, size_t count)
+size_t __clear_user(void __user *to, size_t size)
+{
+	if (static_key_true(&have_mvcos))
+		return clear_user_mvcos(to, size);
+	return clear_user_pt(to, size);
+}
+EXPORT_SYMBOL(__clear_user);
+
+static inline size_t strnlen_user_mvcos(const char __user *src, size_t count)
 {
 	size_t done, len, offset, len_str;
 	char buf[256];
@@ -164,10 +202,18 @@ static size_t strnlen_user_mvcos(const char __user *src, size_t count)
 	return done + 1;
 }
 
-static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
-				      size_t count)
+size_t __strnlen_user(const char __user *src, size_t count)
 {
-	size_t done, len, offset, len_str;
+	if (static_key_true(&have_mvcos))
+		return strnlen_user_mvcos(src, count);
+	return strnlen_user_pt(src, count);
+}
+EXPORT_SYMBOL(__strnlen_user);
+
+static inline size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
+					     size_t count)
+{
+	unsigned long done, len, offset, len_str;
 
 	if (unlikely(!count))
 		return 0;
@@ -185,13 +231,18 @@ static size_t strncpy_from_user_mvcos(char *dst, const char __user *src,
 	return done;
 }
 
-struct uaccess_ops uaccess_mvcos = {
-	.copy_from_user = copy_from_user_mvcos,
-	.copy_to_user = copy_to_user_mvcos,
-	.copy_in_user = copy_in_user_mvcos,
-	.clear_user = clear_user_mvcos,
-	.strnlen_user = strnlen_user_mvcos,
-	.strncpy_from_user = strncpy_from_user_mvcos,
-	.futex_atomic_op = futex_atomic_op_pt,
-	.futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt,
-};
+long __strncpy_from_user(char *dst, const char __user *src, long count)
+{
+	if (static_key_true(&have_mvcos))
+		return strncpy_from_user_mvcos(dst, src, count);
+	return strncpy_from_user_pt(dst, src, count);
+}
+EXPORT_SYMBOL(__strncpy_from_user);
+
+static int __init uaccess_init(void)
+{
+	if (!MACHINE_HAS_MVCOS)
+		static_key_slow_dec(&have_mvcos);
+	return 0;
+}
+early_initcall(uaccess_init);
diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 2fa696b..b49c3a4 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -211,7 +211,7 @@ fault:
 	return 0;
 }
 
-static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
+size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
 {
 	size_t rc;
 
@@ -223,14 +223,14 @@ static size_t copy_from_user_pt(void *to, const void __user *from, size_t n)
 	return rc;
 }
 
-static size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
+size_t copy_to_user_pt(void __user *to, const void *from, size_t n)
 {
 	if (segment_eq(get_fs(), KERNEL_DS))
 		return copy_in_kernel(to, (void __user *) from, n);
 	return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
 }
 
-static size_t clear_user_pt(void __user *to, size_t n)
+size_t clear_user_pt(void __user *to, size_t n)
 {
 	void *zpage = (void *) empty_zero_page;
 	long done, size, ret;
@@ -253,7 +253,7 @@ static size_t clear_user_pt(void __user *to, size_t n)
 	return 0;
 }
 
-static size_t strnlen_user_pt(const char __user *src, size_t count)
+size_t strnlen_user_pt(const char __user *src, size_t count)
 {
 	unsigned long uaddr = (unsigned long) src;
 	struct mm_struct *mm = current->mm;
@@ -289,8 +289,7 @@ fault:
 	goto retry;
 }
 
-static size_t strncpy_from_user_pt(char *dst, const char __user *src,
-				   size_t count)
+size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count)
 {
 	size_t done, len, offset, len_str;
 
@@ -315,8 +314,7 @@ static size_t strncpy_from_user_pt(char *dst, const char __user *src,
 	return done;
 }
 
-static size_t copy_in_user_pt(void __user *to, const void __user *from,
-			      size_t n)
+size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n)
 {
 	struct mm_struct *mm = current->mm;
 	unsigned long offset_max, uaddr, done, size, error_code;
@@ -411,7 +409,7 @@ static int __futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
 	return ret;
 }
 
-int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old)
+int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old)
 {
 	int ret;
 
@@ -449,8 +447,8 @@ static int __futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
 	return ret;
 }
 
-int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
-			    u32 oldval, u32 newval)
+int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
+				  u32 oldval, u32 newval)
 {
 	int ret;
 
@@ -471,14 +469,3 @@ int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr,
 	put_page(virt_to_page(uaddr));
 	return ret;
 }
-
-struct uaccess_ops uaccess_pt = {
-	.copy_from_user		= copy_from_user_pt,
-	.copy_to_user		= copy_to_user_pt,
-	.copy_in_user		= copy_in_user_pt,
-	.clear_user		= clear_user_pt,
-	.strnlen_user		= strnlen_user_pt,
-	.strncpy_from_user	= strncpy_from_user_pt,
-	.futex_atomic_op	= futex_atomic_op_pt,
-	.futex_atomic_cmpxchg	= futex_atomic_cmpxchg_pt,
-};