summaryrefslogtreecommitdiffstats
path: root/arch/x86/lib
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-10-13 18:14:50 +0200
committerLinus Torvalds <torvalds@linux-foundation.org>2014-10-13 18:14:50 +0200
commit197fe6b0e6843b6859c6a1436ff19e3c444c0502 (patch)
tree0aca5adb302db005bbb5e95f9c7905f279769a3c /arch/x86/lib
parentfaafcba3b5e15999cf75d5c5a513ac8e47e2545f (diff)
parentf74954f01ec9bb2004bcc24f247d1f26f1063ad2 (diff)
downloadop-kernel-dev-197fe6b0e6843b6859c6a1436ff19e3c444c0502.zip
op-kernel-dev-197fe6b0e6843b6859c6a1436ff19e3c444c0502.tar.gz
Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 asm updates from Ingo Molnar: "The changes in this cycle were: - Speed up the x86 __preempt_schedule() implementation - Fix/improve low level asm code debug info annotations" * 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86: Unwind-annotate thunk_32.S x86: Improve cmpxchg8b_emu.S x86: Improve cmpxchg16b_emu.S x86/lib/Makefile: Remove the unnecessary "+= thunk_64.o" x86: Speed up ___preempt_schedule*() by using THUNK helpers
Diffstat (limited to 'arch/x86/lib')
-rw-r--r--arch/x86/lib/Makefile2
-rw-r--r--arch/x86/lib/cmpxchg16b_emu.S32
-rw-r--r--arch/x86/lib/cmpxchg8b_emu.S20
-rw-r--r--arch/x86/lib/thunk_32.S41
-rw-r--r--arch/x86/lib/thunk_64.S7
5 files changed, 61 insertions, 41 deletions
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 7ef9a30..db92793 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -38,7 +38,7 @@ endif
else
obj-y += iomap_copy_64.o
lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o
- lib-y += thunk_64.o clear_page_64.o copy_page_64.o
+ lib-y += clear_page_64.o copy_page_64.o
lib-y += memmove_64.o memset_64.o
lib-y += copy_user_64.o copy_user_nocache_64.o
lib-y += cmpxchg16b_emu.o
diff --git a/arch/x86/lib/cmpxchg16b_emu.S b/arch/x86/lib/cmpxchg16b_emu.S
index 1e572c5..40a1725 100644
--- a/arch/x86/lib/cmpxchg16b_emu.S
+++ b/arch/x86/lib/cmpxchg16b_emu.S
@@ -6,15 +6,8 @@
*
*/
#include <linux/linkage.h>
-#include <asm/alternative-asm.h>
-#include <asm/frame.h>
#include <asm/dwarf2.h>
-
-#ifdef CONFIG_SMP
-#define SEG_PREFIX %gs:
-#else
-#define SEG_PREFIX
-#endif
+#include <asm/percpu.h>
.text
@@ -39,24 +32,25 @@ CFI_STARTPROC
# *atomic* on a single cpu (as provided by the this_cpu_xx class of
# macros).
#
-this_cpu_cmpxchg16b_emu:
- pushf
+ pushfq_cfi
cli
- cmpq SEG_PREFIX(%rsi), %rax
- jne not_same
- cmpq SEG_PREFIX 8(%rsi), %rdx
- jne not_same
+ cmpq PER_CPU_VAR((%rsi)), %rax
+ jne .Lnot_same
+ cmpq PER_CPU_VAR(8(%rsi)), %rdx
+ jne .Lnot_same
- movq %rbx, SEG_PREFIX(%rsi)
- movq %rcx, SEG_PREFIX 8(%rsi)
+ movq %rbx, PER_CPU_VAR((%rsi))
+ movq %rcx, PER_CPU_VAR(8(%rsi))
- popf
+ CFI_REMEMBER_STATE
+ popfq_cfi
mov $1, %al
ret
- not_same:
- popf
+ CFI_RESTORE_STATE
+.Lnot_same:
+ popfq_cfi
xor %al,%al
ret
diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S
index 828cb71..b4807fce 100644
--- a/arch/x86/lib/cmpxchg8b_emu.S
+++ b/arch/x86/lib/cmpxchg8b_emu.S
@@ -7,11 +7,8 @@
*/
#include <linux/linkage.h>
-#include <asm/alternative-asm.h>
-#include <asm/frame.h>
#include <asm/dwarf2.h>
-
.text
/*
@@ -30,27 +27,28 @@ CFI_STARTPROC
# set the whole ZF thing (caller will just compare
# eax:edx with the expected value)
#
-cmpxchg8b_emu:
- pushfl
+ pushfl_cfi
cli
cmpl (%esi), %eax
- jne not_same
+ jne .Lnot_same
cmpl 4(%esi), %edx
- jne half_same
+ jne .Lhalf_same
movl %ebx, (%esi)
movl %ecx, 4(%esi)
- popfl
+ CFI_REMEMBER_STATE
+ popfl_cfi
ret
- not_same:
+ CFI_RESTORE_STATE
+.Lnot_same:
movl (%esi), %eax
- half_same:
+.Lhalf_same:
movl 4(%esi), %edx
- popfl
+ popfl_cfi
ret
CFI_ENDPROC
diff --git a/arch/x86/lib/thunk_32.S b/arch/x86/lib/thunk_32.S
index 28f85c91..e28cdaf 100644
--- a/arch/x86/lib/thunk_32.S
+++ b/arch/x86/lib/thunk_32.S
@@ -6,25 +6,46 @@
*/
#include <linux/linkage.h>
#include <asm/asm.h>
+ #include <asm/dwarf2.h>
-#ifdef CONFIG_TRACE_IRQFLAGS
/* put return address in eax (arg1) */
- .macro thunk_ra name,func
+ .macro THUNK name, func, put_ret_addr_in_eax=0
.globl \name
\name:
- pushl %eax
- pushl %ecx
- pushl %edx
+ CFI_STARTPROC
+ pushl_cfi %eax
+ CFI_REL_OFFSET eax, 0
+ pushl_cfi %ecx
+ CFI_REL_OFFSET ecx, 0
+ pushl_cfi %edx
+ CFI_REL_OFFSET edx, 0
+
+ .if \put_ret_addr_in_eax
/* Place EIP in the arg1 */
movl 3*4(%esp), %eax
+ .endif
+
call \func
- popl %edx
- popl %ecx
- popl %eax
+ popl_cfi %edx
+ CFI_RESTORE edx
+ popl_cfi %ecx
+ CFI_RESTORE ecx
+ popl_cfi %eax
+ CFI_RESTORE eax
ret
+ CFI_ENDPROC
_ASM_NOKPROBE(\name)
.endm
- thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller
- thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller
+#ifdef CONFIG_TRACE_IRQFLAGS
+ THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
+ THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
+#endif
+
+#ifdef CONFIG_PREEMPT
+ THUNK ___preempt_schedule, preempt_schedule
+#ifdef CONFIG_CONTEXT_TRACKING
+ THUNK ___preempt_schedule_context, preempt_schedule_context
#endif
+#endif
+
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 92d9fea..b30b5eb 100644
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -38,6 +38,13 @@
THUNK lockdep_sys_exit_thunk,lockdep_sys_exit
#endif
+#ifdef CONFIG_PREEMPT
+ THUNK ___preempt_schedule, preempt_schedule
+#ifdef CONFIG_CONTEXT_TRACKING
+ THUNK ___preempt_schedule_context, preempt_schedule_context
+#endif
+#endif
+
/* SAVE_ARGS below is used only for the .cfi directives it contains. */
CFI_STARTPROC
SAVE_ARGS
OpenPOWER on IntegriCloud