From 53e857f30867918b3618d8e18902e63291946ef4 Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Mon, 10 Sep 2012 13:00:09 +0200 Subject: s390/mm,tlb: race of lazy TLB flush vs. recreation of TLB entries Git commit 050eef364ad70059 "[S390] fix tlb flushing vs. concurrent /proc accesses" introduced the attach counter to avoid using the mm_users value to decide between IPTE for every PTE and lazy TLB flushing with IDTE. That fixed the problem with mm_users but it introduced another subtle race, fortunately one that is very hard to hit. The background is the requirement of the architecture that a valid PTE may not be changed while it can be used concurrently by another cpu. The decision between IPTE and lazy TLB flushing needs to be done while the PTE is still valid. Now if the virtual cpu is temporarily stopped after the decision to use lazy TLB flushing but before the invalid bit of the PTE has been set, another cpu can attach the mm, find that flush_mm is set, do the IDTE, return to userspace, and recreate a TLB that uses the PTE in question. When the first, stopped cpu continues it will change the PTE while it is attached on another cpu. The first cpu will do another IDTE shortly after the modification of the PTE which makes the race window quite short. To fix this race the CPU that wants to attach the address space of a user space thread needs to wait for the end of the PTE modification. The number of concurrent TLB flushers for an mm is tracked in the upper 16 bits of the attach_count and finish_arch_post_lock_switch is used to wait for the end of the flush operation if required. Signed-off-by: Martin Schwidefsky --- arch/s390/kernel/entry.S | 9 ++++++--- arch/s390/kernel/entry64.S | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) (limited to 'arch/s390/kernel') diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 0dc2b6d..526d373 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -43,6 +43,7 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_MCCK_PENDING) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) +_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT) STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER STACK_SIZE = 1 << STACK_SHIFT @@ -159,10 +160,12 @@ ENTRY(__switch_to) lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next l %r15,__THREAD_ksp(%r3) # load kernel stack of next - tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending? + lhi %r6,_TIF_TRANSFER # transfer TIF bits + n %r6,__TI_flags(%r4) # isolate TIF bits jz 0f - ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev - oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next + o %r6,__TI_flags(%r5) # set TIF bits of next + st %r6,__TI_flags(%r5) + ni __TI_flags+3(%r4),255-_TIF_TRANSFER # clear TIF bits of prev 0: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task br %r14 diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 384e609..e09dbe5 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -48,6 +48,7 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_MCCK_PENDING) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) +_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT) #define BASED(name) name-system_call(%r13) @@ -189,10 +190,12 @@ ENTRY(__switch_to) lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 mvc __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next lg %r15,__THREAD_ksp(%r3) # load kernel stack of next - tm __TI_flags+7(%r4),_TIF_MCCK_PENDING # machine check pending? + llill %r6,_TIF_TRANSFER # transfer TIF bits + ng %r6,__TI_flags(%r4) # isolate TIF bits jz 0f - ni __TI_flags+7(%r4),255-_TIF_MCCK_PENDING # clear flag in prev - oi __TI_flags+7(%r5),_TIF_MCCK_PENDING # set it in next + og %r6,__TI_flags(%r5) # set TIF bits of next + stg %r6,__TI_flags(%r5) + ni __TI_flags+7(%r4),255-_TIF_TRANSFER # clear TIF bits of prev 0: lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task br %r14 -- cgit v1.1