Close a race.

The RAS implementation would set the end address, then the start
address.  These were used by the kernel to restart a RAS sequence if
it was interrupted.  When the thread switching code ran, it would
check these values and adjust the PC and clear them if it did.

However, there's a small flaw in this scheme.  Thread T1, sets the end
address and gets preempted.  Thread T2 runs and also does a RAS
operation.  This resets end to zero.  Thread T1 now runs again and
sets start and then begins the RAS sequence, but is preempted before
the RAS sequence executes its last instruction.  The kernel code that
would ordinarily restart the RAS sequence doesn't because the PC isn't
between start and 0, so the PC isn't set to the start of the sequence.
So when T1 is resumed again, it is at the wrong location for RAS to
produce the correct results.  This causes the wrong results for the
atomic sequence.

The window for the first race is 3 instructions.  The window for the
second race is 5-10 instructions depending on the atomic operation.
This makes this failure fairly rare and hard to reproduce.

Mutexs are implemented in libthr using atomic operations.  When the
above race would occur, a lock could get stuck locked, causing many
downstream problems, as you might expect.

Also, make sure to reset the start and end address when doing a syscall, or
a malicious process could set them before doing a syscall.

Reviewed by: imp, ups (thanks guys)
Pointy hat to:	cognet
MFC After:	3 days

1 files changed, 41 insertions, 18 deletions

diff --git a/sys/arm/include/atomic.h b/sys/arm/include/atomic.h
index 79e10eb..bed5a72 100644
--- a/sys/arm/include/atomic.h
+++ b/sys/arm/include/atomic.h
@@ -148,17 +148,21 @@ atomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_in
 	register int done, ras_start;
 
 	__asm __volatile("1:\n"
-	    "mov	%0, #0xe0000008\n"
-	    "adr	%1, 2f\n"
-	    "str	%1, [%0]\n"
 	    "adr	%1, 1b\n"
 	    "mov	%0, #0xe0000004\n"
 	    "str	%1, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "adr	%1, 2f\n"
+	    "str	%1, [%0]\n"
 	    "ldr	%1, [%2]\n"
 	    "cmp	%1, %3\n"
 	    "streq	%4, [%2]\n"
 	    "2:\n"
 	    "mov	%1, #0\n"
+	    "mov	%0, #0xe0000004\n"
+	    "str	%1, [%0]\n"
+	    "mov	%1, #0xffffffff\n"
+	    "mov	%0, #0xe0000008\n"
 	    "str	%1, [%0]\n"
 	    "moveq	%1, #1\n"
 	    "movne	%1, #0\n"
@@ -173,18 +177,22 @@ atomic_add_32(volatile u_int32_t *p, u_int32_t val)
 	int ras_start, start;
 
 	__asm __volatile("1:\n"
-	    "mov	%0, #0xe0000008\n"
-	    "adr	%1, 2f\n"
-	    "str	%1, [%0]\n"
 	    "adr	%1, 1b\n"
 	    "mov	%0, #0xe0000004\n"
 	    "str	%1, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "adr	%1, 2f\n"
+	    "str	%1, [%0]\n"
 	    "ldr	%1, [%2]\n"
 	    "add	%1, %1, %3\n"
 	    "str	%1, [%2]\n"
 	    "2:\n"
+	    "mov	%0, #0xe0000004\n"
 	    "mov	%1, #0\n"
 	    "str	%1, [%0]\n"
+	    "mov	%1, #0xffffffff\n"
+	    "mov	%0, #0xe0000008\n"
+	    "str	%1, [%0]\n"
 	    : "=r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
 	    : : "memory");
 }
@@ -195,18 +203,22 @@ atomic_subtract_32(volatile u_int32_t *p, u_int32_t val)
 	int ras_start, start;
 
 	__asm __volatile("1:\n"
-	    "mov	%0, #0xe0000008\n"
-	    "adr	%1, 2f\n"
-	    "str	%1, [%0]\n"
 	    "adr	%1, 1b\n"
 	    "mov	%0, #0xe0000004\n"
 	    "str	%1, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "adr	%1, 2f\n"
+	    "str	%1, [%0]\n"
 	    "ldr	%1, [%2]\n"
 	    "sub	%1, %1, %3\n"
 	    "str	%1, [%2]\n"
 	    "2:\n"
+	    "mov	%0, #0xe0000004\n"
 	    "mov	%1, #0\n"
 	    "str	%1, [%0]\n"
+	    "mov	%1, #0xffffffff\n"
+	    "mov	%0, #0xe0000008\n"
+	    "str	%1, [%0]\n"
 
 	    : "=r" (ras_start), "=r" (start), "+r" (p), "+r" (val)
 	    : : "memory");
@@ -218,18 +230,22 @@ atomic_set_32(volatile uint32_t *address, uint32_t setmask)
 	int ras_start, start;
 
 	__asm __volatile("1:\n"
-	    "mov	%0, #0xe0000008\n"
-	    "adr	%1, 2f\n"
-	    "str	%1, [%0]\n"
 	    "adr	%1, 1b\n"
 	    "mov	%0, #0xe0000004\n"
 	    "str	%1, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "adr	%1, 2f\n"
+	    "str	%1, [%0]\n"
 	    "ldr	%1, [%2]\n"
 	    "orr	%1, %1, %3\n"
 	    "str	%1, [%2]\n"
 	    "2:\n"
+	    "mov	%0, #0xe0000004\n"
 	    "mov	%1, #0\n"
 	    "str	%1, [%0]\n"
+	    "mov	%1, #0xffffffff\n"
+	    "mov	%0, #0xe0000008\n"
+	    "str	%1, [%0]\n"
 
 	    : "=r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask)
 	    : : "memory");
@@ -241,18 +257,22 @@ atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
 	int ras_start, start;
 
 	__asm __volatile("1:\n"
-	    "mov	%0, #0xe0000008\n"
-	    "adr	%1, 2f\n"
-	    "str	%1, [%0]\n"
 	    "adr	%1, 1b\n"
 	    "mov	%0, #0xe0000004\n"
 	    "str	%1, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "adr	%1, 2f\n"
+	    "str	%1, [%0]\n"
 	    "ldr	%1, [%2]\n"
 	    "bic	%1, %1, %3\n"
 	    "str	%1, [%2]\n"
 	    "2:\n"
+	    "mov	%0, #0xe0000004\n"
 	    "mov	%1, #0\n"
 	    "str	%1, [%0]\n"
+	    "mov	%1, #0xffffffff\n"
+	    "mov	%0, #0xe0000008\n"
+	    "str	%1, [%0]\n"
 	    : "=r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask)
 	    : : "memory");
 
@@ -264,12 +284,12 @@ atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
 	uint32_t ras_start, start;
 
 	__asm __volatile("1:\n"
-	    "mov	%0, #0xe0000008\n"
-	    "adr	%1, 2f\n"
-	    "str	%1, [%0]\n"
 	    "adr	%1, 1b\n"
 	    "mov	%0, #0xe0000004\n"
 	    "str	%1, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "adr	%1, 2f\n"
+	    "str	%1, [%0]\n"
 	    "ldr	%1, [%2]\n"
 	    "add	%0, %1, %3\n"
 	    "str	%0, [%2]\n"
@@ -277,6 +297,9 @@ atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
 	    "mov	%0, #0xe0000004\n"
 	    "mov	%3, #0\n"
 	    "str	%3, [%0]\n"
+	    "mov	%0, #0xe0000008\n"
+	    "mov	%3, #0xffffffff\n"
+	    "str	%3, [%0]\n"
 	    : "=r" (ras_start), "=r" (start), "+r" (p), "+r" (v)
 	    : : "memory");
 	return (start);