diff options
| -rw-r--r-- | arch/s390/include/asm/futex.h | 13 | ||||
| -rw-r--r-- | arch/s390/include/asm/uaccess.h | 148 | ||||
| -rw-r--r-- | arch/s390/kernel/setup.c | 9 | ||||
| -rw-r--r-- | arch/s390/lib/Makefile | 3 | ||||
| -rw-r--r-- | arch/s390/lib/uaccess.h | 8 | ||||
| -rw-r--r-- | arch/s390/lib/uaccess_mvcos.c | 89 | ||||
| -rw-r--r-- | arch/s390/lib/uaccess_pt.c | 31 |
7 files changed, 152 insertions, 149 deletions
diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index 51bcaa0..fda46bd 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -5,7 +5,10 @@ #include <linux/uaccess.h> #include <asm/errno.h> -static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) +int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval); +int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old); + +static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr) { int op = (encoded_op >> 28) & 7; int cmp = (encoded_op >> 24) & 15; @@ -17,7 +20,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) oparg = 1 << oparg; pagefault_disable(); - ret = uaccess.futex_atomic_op(op, uaddr, oparg, &oldval); + ret = __futex_atomic_op_inuser(op, uaddr, oparg, &oldval); pagefault_enable(); if (!ret) { @@ -34,10 +37,4 @@ static inline int futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) return ret; } -static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, - u32 oldval, u32 newval) -{ - return uaccess.futex_atomic_cmpxchg(uval, uaddr, oldval, newval); -} - #endif /* _ASM_S390_FUTEX_H */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 7319963..49885a5 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -92,33 +92,58 @@ static inline unsigned long extable_fixup(const struct exception_table_entry *x) #define ARCH_HAS_SORT_EXTABLE #define ARCH_HAS_SEARCH_EXTABLE -struct uaccess_ops { - size_t (*copy_from_user)(void *, const void __user *, size_t); - size_t (*copy_to_user)(void __user *, const void *, size_t); - size_t (*copy_in_user)(void __user *, const void __user *, size_t); - size_t (*clear_user)(void __user *, size_t); - size_t (*strnlen_user)(const char __user *, size_t); - size_t (*strncpy_from_user)(char *, const char __user *, size_t); - int (*futex_atomic_op)(int op, u32 __user *, int oparg, int *old); - int (*futex_atomic_cmpxchg)(u32 *, u32 __user *, u32 old, u32 new); -}; +int __handle_fault(unsigned long, unsigned long, int); -extern struct uaccess_ops uaccess; -extern struct uaccess_ops uaccess_mvcos; -extern struct uaccess_ops uaccess_pt; +/** + * __copy_from_user: - Copy a block of data from user space, with less checking. + * @to: Destination address, in kernel space. + * @from: Source address, in user space. + * @n: Number of bytes to copy. + * + * Context: User context only. This function may sleep. + * + * Copy data from user space to kernel space. Caller must check + * the specified block with access_ok() before calling this function. + * + * Returns number of bytes that could not be copied. + * On success, this will be zero. + * + * If some data could not be copied, this function will pad the copied + * data to the requested size using zero bytes. + */ +size_t __must_check __copy_from_user(void *to, const void __user *from, + size_t n); + +/** + * __copy_to_user: - Copy a block of data into user space, with less checking. + * @to: Destination address, in user space. + * @from: Source address, in kernel space. + * @n: Number of bytes to copy. + * + * Context: User context only. This function may sleep. + * + * Copy data from kernel space to user space. Caller must check + * the specified block with access_ok() before calling this function. + * + * Returns number of bytes that could not be copied. + * On success, this will be zero. + */ +unsigned long __must_check __copy_to_user(void __user *to, const void *from, + unsigned long n); -extern int __handle_fault(unsigned long, unsigned long, int); +#define __copy_to_user_inatomic __copy_to_user +#define __copy_from_user_inatomic __copy_from_user static inline int __put_user_fn(void *x, void __user *ptr, size_t size) { - size = uaccess.copy_to_user(ptr, x, size); - return size ? -EFAULT : size; + size = __copy_to_user(ptr, x, size); + return size ? -EFAULT : 0; } static inline int __get_user_fn(void *x, const void __user *ptr, size_t size) { - size = uaccess.copy_from_user(x, ptr, size); - return size ? -EFAULT : size; + size = __copy_from_user(x, ptr, size); + return size ? -EFAULT : 0; } /* @@ -152,7 +177,7 @@ static inline int __get_user_fn(void *x, const void __user *ptr, size_t size) }) -extern int __put_user_bad(void) __attribute__((noreturn)); +int __put_user_bad(void) __attribute__((noreturn)); #define __get_user(x, ptr) \ ({ \ @@ -200,35 +225,12 @@ extern int __put_user_bad(void) __attribute__((noreturn)); __get_user(x, ptr); \ }) -extern int __get_user_bad(void) __attribute__((noreturn)); +int __get_user_bad(void) __attribute__((noreturn)); #define __put_user_unaligned __put_user #define __get_user_unaligned __get_user /** - * __copy_to_user: - Copy a block of data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -static inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) -{ - return uaccess.copy_to_user(to, from, n); -} - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - -/** * copy_to_user: - Copy a block of data into user space. * @to: Destination address, in user space. * @from: Source address, in kernel space. @@ -248,30 +250,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n) return __copy_to_user(to, from, n); } -/** - * __copy_from_user: - Copy a block of data from user space, with less checking. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep. - * - * Copy data from user space to kernel space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -static inline unsigned long __must_check -__copy_from_user(void *to, const void __user *from, unsigned long n) -{ - return uaccess.copy_from_user(to, from, n); -} - -extern void copy_from_user_overflow(void) +void copy_from_user_overflow(void) #ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS __compiletime_warning("copy_from_user() buffer size is not provably correct") #endif @@ -306,11 +285,8 @@ copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_from_user(to, from, n); } -static inline unsigned long __must_check -__copy_in_user(void __user *to, const void __user *from, unsigned long n) -{ - return uaccess.copy_in_user(to, from, n); -} +unsigned long __must_check +__copy_in_user(void __user *to, const void __user *from, unsigned long n); static inline unsigned long __must_check copy_in_user(void __user *to, const void __user *from, unsigned long n) @@ -322,18 +298,22 @@ copy_in_user(void __user *to, const void __user *from, unsigned long n) /* * Copy a null terminated string from userspace. */ + +long __strncpy_from_user(char *dst, const char __user *src, long count); + static inline long __must_check strncpy_from_user(char *dst, const char __user *src, long count) { might_fault(); - return uaccess.strncpy_from_user(dst, src, count); + return __strncpy_from_user(dst, src, count); } -static inline unsigned long -strnlen_user(const char __user * src, unsigned long n) +size_t __must_check __strnlen_user(const char __user *src, size_t count); + +static inline size_t strnlen_user(const char __user *src, size_t n) { might_fault(); - return uaccess.strnlen_user(src, n); + return __strnlen_user(src, n); } /** @@ -355,21 +335,15 @@ strnlen_user(const char __user * src, unsigned long n) /* * Zero Userspace */ +size_t __must_check __clear_user(void __user *to, size_t size); -static inline unsigned long __must_check -__clear_user(void __user *to, unsigned long n) -{ - return uaccess.clear_user(to, n); -} - -static inline unsigned long __must_check -clear_user(void __user *to, unsigned long n) +static inline size_t __must_check clear_user(void __user *to, size_t n) { might_fault(); - return uaccess.clear_user(to, n); + return __clear_user(to, n); } -extern int copy_to_user_real(void __user *dest, void *src, size_t count); -extern int copy_from_user_real(void *dest, void __user *src, size_t count); +int copy_to_user_real(void __user *dest, void *src, size_t count); +int copy_from_user_real(void *dest, void __user *src, size_t count); #endif /* __S390_UACCESS_H */ diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 09e2f46..91ea009 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -47,7 +47,6 @@ #include <linux/compat.h> #include <asm/ipl.h> -#include <asm/uaccess.h> #include <asm/facility.h> #include <asm/smp.h> #include <asm/mmu_context.h> @@ -65,12 +64,6 @@ #include "entry.h" /* - * User copy operations. - */ -struct uaccess_ops uaccess; -EXPORT_SYMBOL(uaccess); - -/* * Machine setup.. */ unsigned int console_mode = 0; @@ -1009,8 +1002,6 @@ void __init setup_arch(char **cmdline_p) init_mm.end_data = (unsigned long) &_edata; init_mm.brk = (unsigned long) &_end; - uaccess = MACHINE_HAS_MVCOS ? uaccess_mvcos : uaccess_pt; - parse_early_param(); detect_memory_layout(memory_chunk, memory_end); os_info_init(); diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index b068729..e3fffe1 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -2,8 +2,7 @@ # Makefile for s390-specific library files.. # -lib-y += delay.o string.o uaccess_pt.o find.o +lib-y += delay.o string.o uaccess_pt.o uaccess_mvcos.o find.o obj-$(CONFIG_32BIT) += div64.o qrnnd.o ucmpdi2.o mem32.o obj-$(CONFIG_64BIT) += mem64.o -lib-$(CONFIG_64BIT) += uaccess_mvcos.o lib-$(CONFIG_SMP) += spinlock.o diff --git a/arch/s390/lib/uaccess.h b/arch/s390/lib/uaccess.h index b1a2217..e5b9c924 100644 --- a/arch/s390/lib/uaccess.h +++ b/arch/s390/lib/uaccess.h @@ -6,7 +6,11 @@ #ifndef __ARCH_S390_LIB_UACCESS_H #define __ARCH_S390_LIB_UACCESS_H -extern int futex_atomic_op_pt(int, u32 __user *, int, int *); -extern int futex_atomic_cmpxchg_pt(u32 *, u32 __user *, u32, u32); +size_t copy_from_user_pt(void *to, const void __user *from, size_t n); +size_t copy_to_user_pt(void __user *to, const void *from, size_t n); +size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n); +size_t clear_user_pt(void __user *to, size_t n); +size_t strnlen_user_pt(const char __user *src, size_t count); +size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count); #endif /* __ARCH_S390_LIB_UACCESS_H */ diff --git a/arch/s390/lib/uaccess_mvcos.c b/arch/s390/lib/uaccess_mvcos.c index 95123f5..66f35e1 100644 --- a/arch/s390/lib/uaccess_mvcos.c +++ b/arch/s390/lib/uaccess_mvcos.c @@ -6,7 +6,9 @@ * Gerald Schaefer (gerald.schaefer@de.ibm.com) */ +#include <linux/jump_label.h> #include <linux/errno.h> +#include <linux/init.h> #include <linux/mm.h> #include <asm/uaccess.h> #include <asm/futex.h> @@ -26,7 +28,10 @@ #define SLR "slgr" #endif -static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size) +static struct static_key have_mvcos = STATIC_KEY_INIT_TRUE; + +static inline size_t copy_from_user_mvcos(void *x, const void __user *ptr, + size_t size) { register unsigned long reg0 asm("0") = 0x81UL; unsigned long tmp1, tmp2; @@ -65,7 +70,16 @@ static size_t copy_from_user_mvcos(void *x, const void __user *ptr, size_t size) return size; } -static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size) +size_t __copy_from_user(void *to, const void __user *from, size_t n) +{ + if (static_key_true(&have_mvcos)) + return copy_from_user_mvcos(to, from, n); + return copy_from_user_pt(to, from, n); +} +EXPORT_SYMBOL(__copy_from_user); + +static inline size_t copy_to_user_mvcos(void __user *ptr, const void *x, + size_t size) { register unsigned long reg0 asm("0") = 0x810000UL; unsigned long tmp1, tmp2; @@ -94,8 +108,16 @@ static size_t copy_to_user_mvcos(void __user *ptr, const void *x, size_t size) return size; } -static size_t copy_in_user_mvcos(void __user *to, const void __user *from, - size_t size) +size_t __copy_to_user(void __user *to, const void *from, size_t n) +{ + if (static_key_true(&have_mvcos)) + return copy_to_user_mvcos(to, from, n); + return copy_to_user_pt(to, from, n); +} +EXPORT_SYMBOL(__copy_to_user); + +static inline size_t copy_in_user_mvcos(void __user *to, const void __user *from, + size_t size) { register unsigned long reg0 asm("0") = 0x810081UL; unsigned long tmp1, tmp2; @@ -117,7 +139,15 @@ static size_t copy_in_user_mvcos(void __user *to, const void __user *from, return size; } -static size_t clear_user_mvcos(void __user *to, size_t size) +size_t __copy_in_user(void __user *to, const void __user *from, size_t n) +{ + if (static_key_true(&have_mvcos)) + return copy_in_user_mvcos(to, from, n); + return copy_in_user_pt(to, from, n); +} +EXPORT_SYMBOL(__copy_in_user); + +static inline size_t clear_user_mvcos(void __user *to, size_t size) { register unsigned long reg0 asm("0") = 0x810000UL; unsigned long tmp1, tmp2; @@ -145,7 +175,15 @@ static size_t clear_user_mvcos(void __user *to, size_t size) return size; } -static size_t strnlen_user_mvcos(const char __user *src, size_t count) +size_t __clear_user(void __user *to, size_t size) +{ + if (static_key_true(&have_mvcos)) + return clear_user_mvcos(to, size); + return clear_user_pt(to, size); +} +EXPORT_SYMBOL(__clear_user); + +static inline size_t strnlen_user_mvcos(const char __user *src, size_t count) { size_t done, len, offset, len_str; char buf[256]; @@ -164,10 +202,18 @@ static size_t strnlen_user_mvcos(const char __user *src, size_t count) return done + 1; } -static size_t strncpy_from_user_mvcos(char *dst, const char __user *src, - size_t count) +size_t __strnlen_user(const char __user *src, size_t count) { - size_t done, len, offset, len_str; + if (static_key_true(&have_mvcos)) + return strnlen_user_mvcos(src, count); + return strnlen_user_pt(src, count); +} +EXPORT_SYMBOL(__strnlen_user); + +static inline size_t strncpy_from_user_mvcos(char *dst, const char __user *src, + size_t count) +{ + unsigned long done, len, offset, len_str; if (unlikely(!count)) return 0; @@ -185,13 +231,18 @@ static size_t strncpy_from_user_mvcos(char *dst, const char __user *src, return done; } -struct uaccess_ops uaccess_mvcos = { - .copy_from_user = copy_from_user_mvcos, - .copy_to_user = copy_to_user_mvcos, - .copy_in_user = copy_in_user_mvcos, - .clear_user = clear_user_mvcos, - .strnlen_user = strnlen_user_mvcos, - .strncpy_from_user = strncpy_from_user_mvcos, - .futex_atomic_op = futex_atomic_op_pt, - .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt, -}; +long __strncpy_from_user(char *dst, const char __user *src, long count) +{ + if (static_key_true(&have_mvcos)) + return strncpy_from_user_mvcos(dst, src, count); + return strncpy_from_user_pt(dst, src, count); +} +EXPORT_SYMBOL(__strncpy_from_user); + +static int __init uaccess_init(void) +{ + if (!MACHINE_HAS_MVCOS) + static_key_slow_dec(&have_mvcos); + return 0; +} +early_initcall(uaccess_init); diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c index 2fa696b..b49c3a4 100644 --- a/arch/s390/lib/uaccess_pt.c +++ b/arch/s390/lib/uaccess_pt.c @@ -211,7 +211,7 @@ fault: return 0; } -static size_t copy_from_user_pt(void *to, const void __user *from, size_t n) +size_t copy_from_user_pt(void *to, const void __user *from, size_t n) { size_t rc; @@ -223,14 +223,14 @@ static size_t copy_from_user_pt(void *to, const void __user *from, size_t n) return rc; } -static size_t copy_to_user_pt(void __user *to, const void *from, size_t n) +size_t copy_to_user_pt(void __user *to, const void *from, size_t n) { if (segment_eq(get_fs(), KERNEL_DS)) return copy_in_kernel(to, (void __user *) from, n); return __user_copy_pt((unsigned long) to, (void *) from, n, 1); } -static size_t clear_user_pt(void __user *to, size_t n) +size_t clear_user_pt(void __user *to, size_t n) { void *zpage = (void *) empty_zero_page; long done, size, ret; @@ -253,7 +253,7 @@ static size_t clear_user_pt(void __user *to, size_t n) return 0; } -static size_t strnlen_user_pt(const char __user *src, size_t count) +size_t strnlen_user_pt(const char __user *src, size_t count) { unsigned long uaddr = (unsigned long) src; struct mm_struct *mm = current->mm; @@ -289,8 +289,7 @@ fault: goto retry; } -static size_t strncpy_from_user_pt(char *dst, const char __user *src, - size_t count) +size_t strncpy_from_user_pt(char *dst, const char __user *src, size_t count) { size_t done, len, offset, len_str; @@ -315,8 +314,7 @@ static size_t strncpy_from_user_pt(char *dst, const char __user *src, return done; } -static size_t copy_in_user_pt(void __user *to, const void __user *from, - size_t n) +size_t copy_in_user_pt(void __user *to, const void __user *from, size_t n) { struct mm_struct *mm = current->mm; unsigned long offset_max, uaddr, done, size, error_code; @@ -411,7 +409,7 @@ static int __futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old) return ret; } -int futex_atomic_op_pt(int op, u32 __user *uaddr, int oparg, int *old) +int __futex_atomic_op_inuser(int op, u32 __user *uaddr, int oparg, int *old) { int ret; @@ -449,8 +447,8 @@ static int __futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr, return ret; } -int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr, - u32 oldval, u32 newval) +int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) { int ret; @@ -471,14 +469,3 @@ int futex_atomic_cmpxchg_pt(u32 *uval, u32 __user *uaddr, put_page(virt_to_page(uaddr)); return ret; } - -struct uaccess_ops uaccess_pt = { - .copy_from_user = copy_from_user_pt, - .copy_to_user = copy_to_user_pt, - .copy_in_user = copy_in_user_pt, - .clear_user = clear_user_pt, - .strnlen_user = strnlen_user_pt, - .strncpy_from_user = strncpy_from_user_pt, - .futex_atomic_op = futex_atomic_op_pt, - .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt, -}; |
