Convert kern_umtx.c to use fueword() and casueword().

Also fix some mishandling of suword(9) errors as errno, which resulted
in spurious ERESTART.

Sponsored by:	The FreeBSD Foundation
Tested by:	pho
MFC after:	3 weeks

1 files changed, 346 insertions, 217 deletions

diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index c815e36..58e76bc 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -510,6 +510,15 @@ umtxq_unbusy(struct umtx_key *key)
 		wakeup_one(uc);
 }
 
+static inline void
+umtxq_unbusy_unlocked(struct umtx_key *key)
+{
+
+	umtxq_lock(key);
+	umtxq_unbusy(key);
+	umtxq_unlock(key);
+}
+
 static struct umtxq_queue *
 umtxq_queue_lookup(struct umtx_key *key, int q)
 {
@@ -847,6 +856,7 @@ do_wait(struct thread *td, void *addr, u_long id,
 	struct abs_timeout timo;
 	struct umtx_q *uq;
 	u_long tmp;
+	uint32_t tmp32;
 	int error = 0;
 
 	uq = td->td_umtxq;
@@ -860,18 +870,29 @@ do_wait(struct thread *td, void *addr, u_long id,
 	umtxq_lock(&uq->uq_key);
 	umtxq_insert(uq);
 	umtxq_unlock(&uq->uq_key);
-	if (compat32 == 0)
-		tmp = fuword(addr);
-        else
-		tmp = (unsigned int)fuword32(addr);
+	if (compat32 == 0) {
+		error = fueword(addr, &tmp);
+		if (error != 0)
+			error = EFAULT;
+	} else {
+		error = fueword32(addr, &tmp32);
+		if (error == 0)
+			tmp = tmp32;
+		else
+			error = EFAULT;
+	}
 	umtxq_lock(&uq->uq_key);
-	if (tmp == id)
-		error = umtxq_sleep(uq, "uwait", timeout == NULL ?
-		    NULL : &timo);
-	if ((uq->uq_flags & UQF_UMTXQ) == 0)
-		error = 0;
-	else
+	if (error == 0) {
+		if (tmp == id)
+			error = umtxq_sleep(uq, "uwait", timeout == NULL ?
+			    NULL : &timo);
+		if ((uq->uq_flags & UQF_UMTXQ) == 0)
+			error = 0;
+		else
+			umtxq_remove(uq);
+	} else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
 		umtxq_remove(uq);
+	}
 	umtxq_unlock(&uq->uq_key);
 	umtx_key_release(&uq->uq_key);
 	if (error == ERESTART)
@@ -908,11 +929,11 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
 	struct abs_timeout timo;
 	struct umtx_q *uq;
 	uint32_t owner, old, id;
-	int error = 0;
+	int error, rv;
 
 	id = td->td_tid;
 	uq = td->td_umtxq;
-
+	error = 0;
 	if (timeout != NULL)
 		abs_timeout_init2(&timo, timeout);
 
@@ -921,7 +942,9 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
 	 * can fault on any access.
 	 */
 	for (;;) {
-		owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
+		rv = fueword32(__DEVOLATILE(void *, &m->m_owner), &owner);
+		if (rv == -1)
+			return (EFAULT);
 		if (mode == _UMUTEX_WAIT) {
 			if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
 				return (0);
@@ -929,31 +952,31 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
 			/*
 			 * Try the uncontested case.  This should be done in userland.
 			 */
-			owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
+			rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
+			    &owner, id);
+			/* The address was invalid. */
+			if (rv == -1)
+				return (EFAULT);
 
 			/* The acquire succeeded. */
 			if (owner == UMUTEX_UNOWNED)
 				return (0);
 
-			/* The address was invalid. */
-			if (owner == -1)
-				return (EFAULT);
-
 			/* If no one owns it but it is contested try to acquire it. */
 			if (owner == UMUTEX_CONTESTED) {
-				owner = casuword32(&m->m_owner,
-				    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+				rv = casueword32(&m->m_owner,
+				    UMUTEX_CONTESTED, &owner,
+				    id | UMUTEX_CONTESTED);
+				/* The address was invalid. */
+				if (rv == -1)
+					return (EFAULT);
 
 				if (owner == UMUTEX_CONTESTED)
 					return (0);
 
-				/* The address was invalid. */
-				if (owner == -1)
-					return (EFAULT);
-
-				error = umtxq_check_susp(td);
-				if (error != 0)
-					return (error);
+				rv = umtxq_check_susp(td);
+				if (rv != 0)
+					return (rv);
 
 				/* If this failed the lock has changed, restart. */
 				continue;
@@ -985,10 +1008,11 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags,
 		 * either some one else has acquired the lock or it has been
 		 * released.
 		 */
-		old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
+		rv = casueword32(&m->m_owner, owner, &old,
+		    owner | UMUTEX_CONTESTED);
 
 		/* The address was invalid. */
-		if (old == -1) {
+		if (rv == -1) {
 			umtxq_lock(&uq->uq_key);
 			umtxq_remove(uq);
 			umtxq_unbusy(&uq->uq_key);
@@ -1033,16 +1057,16 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
 	/*
 	 * Make sure we own this mtx.
 	 */
-	owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
-	if (owner == -1)
+	error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+	if (error == -1)
 		return (EFAULT);
 
 	if ((owner & ~UMUTEX_CONTESTED) != id)
 		return (EPERM);
 
 	if ((owner & UMUTEX_CONTESTED) == 0) {
-		old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
-		if (old == -1)
+		error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED);
+		if (error == -1)
 			return (EFAULT);
 		if (old == owner)
 			return (0);
@@ -1064,14 +1088,14 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
 	 * there is zero or one thread only waiting for it.
 	 * Otherwise, it must be marked as contested.
 	 */
-	old = casuword32(&m->m_owner, owner,
-		count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
+	error = casueword32(&m->m_owner, owner, &old,
+	    count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
 	umtxq_lock(&key);
 	umtxq_signal(&key,1);
 	umtxq_unbusy(&key);
 	umtxq_unlock(&key);
 	umtx_key_release(&key);
-	if (old == -1)
+	if (error == -1)
 		return (EFAULT);
 	if (old != owner)
 		return (EINVAL);
@@ -1091,14 +1115,16 @@ do_wake_umutex(struct thread *td, struct umutex *m)
 	int error;
 	int count;
 
-	owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
-	if (owner == -1)
+	error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+	if (error == -1)
 		return (EFAULT);
 
 	if ((owner & ~UMUTEX_CONTESTED) != 0)
 		return (0);
 
-	flags = fuword32(&m->m_flags);
+	error = fueword32(&m->m_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 
 	/* We should only ever be in here for contested locks */
 	if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
@@ -1110,16 +1136,20 @@ do_wake_umutex(struct thread *td, struct umutex *m)
 	count = umtxq_count(&key);
 	umtxq_unlock(&key);
 
-	if (count <= 1)
-		owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
+	if (count <= 1) {
+		error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
+		    UMUTEX_UNOWNED);
+		if (error == -1)
+			error = EFAULT;
+	}
 
 	umtxq_lock(&key);
-	if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
+	if (error == 0 && count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
 		umtxq_signal(&key, 1);
 	umtxq_unbusy(&key);
 	umtxq_unlock(&key);
 	umtx_key_release(&key);
-	return (0);
+	return (error);
 }
 
 /*
@@ -1162,41 +1192,49 @@ do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags)
 	 * any memory.
 	 */
 	if (count > 1) {
-		owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
-		while ((owner & UMUTEX_CONTESTED) ==0) {
-			old = casuword32(&m->m_owner, owner,
-			    owner|UMUTEX_CONTESTED);
+		error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner),
+		    &owner);
+		if (error == -1)
+			error = EFAULT;
+		while (error == 0 && (owner & UMUTEX_CONTESTED) == 0) {
+			error = casueword32(&m->m_owner, owner, &old,
+			    owner | UMUTEX_CONTESTED);
+			if (error == -1) {
+				error = EFAULT;
+				break;
+			}
 			if (old == owner)
 				break;
 			owner = old;
-			if (old == -1)
-				break;
 			error = umtxq_check_susp(td);
 			if (error != 0)
 				break;
 		}
 	} else if (count == 1) {
-		owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
-		while ((owner & ~UMUTEX_CONTESTED) != 0 &&
+		error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner),
+		    &owner);
+		if (error == -1)
+			error = EFAULT;
+		while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 &&
 		       (owner & UMUTEX_CONTESTED) == 0) {
-			old = casuword32(&m->m_owner, owner,
-			    owner|UMUTEX_CONTESTED);
+			error = casueword32(&m->m_owner, owner, &old,
+			    owner | UMUTEX_CONTESTED);
+			if (error == -1) {
+				error = EFAULT;
+				break;
+			}
 			if (old == owner)
 				break;
 			owner = old;
-			if (old == -1)
-				break;
 			error = umtxq_check_susp(td);
 			if (error != 0)
 				break;
 		}
 	}
 	umtxq_lock(&key);
-	if (owner == -1) {
-		error = EFAULT;
+	if (error == EFAULT) {
 		umtxq_signal(&key, INT_MAX);
-	}
-	else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
+	} else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
 		umtxq_signal(&key, 1);
 	umtxq_unbusy(&key);
 	umtxq_unlock(&key);
@@ -1576,7 +1614,7 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
 	struct umtx_q *uq;
 	struct umtx_pi *pi, *new_pi;
 	uint32_t id, owner, old;
-	int error;
+	int error, rv;
 
 	id = td->td_tid;
 	uq = td->td_umtxq;
@@ -1619,7 +1657,12 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
 		/*
 		 * Try the uncontested case.  This should be done in userland.
 		 */
-		owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
+		rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
+		/* The address was invalid. */
+		if (rv == -1) {
+			error = EFAULT;
+			break;
+		}
 
 		/* The acquire succeeded. */
 		if (owner == UMUTEX_UNOWNED) {
@@ -1627,16 +1670,15 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
 			break;
 		}
 
-		/* The address was invalid. */
-		if (owner == -1) {
-			error = EFAULT;
-			break;
-		}
-
 		/* If no one owns it but it is contested try to acquire it. */
 		if (owner == UMUTEX_CONTESTED) {
-			owner = casuword32(&m->m_owner,
-			    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+			rv = casueword32(&m->m_owner,
+			    UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED);
+			/* The address was invalid. */
+			if (rv == -1) {
+				error = EFAULT;
+				break;
+			}
 
 			if (owner == UMUTEX_CONTESTED) {
 				umtxq_lock(&uq->uq_key);
@@ -1647,12 +1689,6 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
 				break;
 			}
 
-			/* The address was invalid. */
-			if (owner == -1) {
-				error = EFAULT;
-				break;
-			}
-
 			error = umtxq_check_susp(td);
 			if (error != 0)
 				break;
@@ -1683,13 +1719,12 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags,
 		 * either some one else has acquired the lock or it has been
 		 * released.
 		 */
-		old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED);
+		rv = casueword32(&m->m_owner, owner, &old,
+		    owner | UMUTEX_CONTESTED);
 
 		/* The address was invalid. */
-		if (old == -1) {
-			umtxq_lock(&uq->uq_key);
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_unlock(&uq->uq_key);
+		if (rv == -1) {
+			umtxq_unbusy_unlocked(&uq->uq_key);
 			error = EFAULT;
 			break;
 		}
@@ -1741,8 +1776,8 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
 	/*
 	 * Make sure we own this mtx.
 	 */
-	owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
-	if (owner == -1)
+	error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+	if (error == -1)
 		return (EFAULT);
 
 	if ((owner & ~UMUTEX_CONTESTED) != id)
@@ -1750,8 +1785,8 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
 
 	/* This should be done in userland */
 	if ((owner & UMUTEX_CONTESTED) == 0) {
-		old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
-		if (old == -1)
+		error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED);
+		if (error == -1)
 			return (EFAULT);
 		if (old == owner)
 			return (0);
@@ -1809,14 +1844,12 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
 	 * there is zero or one thread only waiting for it.
 	 * Otherwise, it must be marked as contested.
 	 */
-	old = casuword32(&m->m_owner, owner,
-		count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
+	error = casueword32(&m->m_owner, owner, &old,
+	    count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
 
-	umtxq_lock(&key);
-	umtxq_unbusy(&key);
-	umtxq_unlock(&key);
+	umtxq_unbusy_unlocked(&key);
 	umtx_key_release(&key);
-	if (old == -1)
+	if (error == -1)
 		return (EFAULT);
 	if (old != owner)
 		return (EINVAL);
@@ -1835,7 +1868,7 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
 	struct umtx_pi *pi;
 	uint32_t ceiling;
 	uint32_t owner, id;
-	int error, pri, old_inherited_pri, su;
+	int error, pri, old_inherited_pri, su, rv;
 
 	id = td->td_tid;
 	uq = td->td_umtxq;
@@ -1853,7 +1886,12 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
 		umtxq_busy(&uq->uq_key);
 		umtxq_unlock(&uq->uq_key);
 
-		ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]);
+		rv = fueword32(&m->m_ceilings[0], &ceiling);
+		if (rv == -1) {
+			error = EFAULT;
+			goto out;
+		}
+		ceiling = RTP_PRIO_MAX - ceiling;
 		if (ceiling > RTP_PRIO_MAX) {
 			error = EINVAL;
 			goto out;
@@ -1874,17 +1912,16 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
 		}
 		mtx_unlock_spin(&umtx_lock);
 
-		owner = casuword32(&m->m_owner,
-		    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
-
-		if (owner == UMUTEX_CONTESTED) {
-			error = 0;
+		rv = casueword32(&m->m_owner,
+		    UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED);
+		/* The address was invalid. */
+		if (rv == -1) {
+			error = EFAULT;
 			break;
 		}
 
-		/* The address was invalid. */
-		if (owner == -1) {
-			error = EFAULT;
+		if (owner == UMUTEX_CONTESTED) {
+			error = 0;
 			break;
 		}
 
@@ -1946,9 +1983,7 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags,
 	}
 
 out:
-	umtxq_lock(&uq->uq_key);
-	umtxq_unbusy(&uq->uq_key);
-	umtxq_unlock(&uq->uq_key);
+	umtxq_unbusy_unlocked(&uq->uq_key);
 	umtx_key_release(&uq->uq_key);
 	return (error);
 }
@@ -1973,8 +2008,8 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags)
 	/*
 	 * Make sure we own this mtx.
 	 */
-	owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
-	if (owner == -1)
+	error = fueword32(__DEVOLATILE(uint32_t *, &m->m_owner), &owner);
+	if (error == -1)
 		return (EFAULT);
 
 	if ((owner & ~UMUTEX_CONTESTED) != id)
@@ -2047,9 +2082,11 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
 	uint32_t save_ceiling;
 	uint32_t owner, id;
 	uint32_t flags;
-	int error;
+	int error, rv;
 
-	flags = fuword32(&m->m_flags);
+	error = fueword32(&m->m_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	if ((flags & UMUTEX_PRIO_PROTECT) == 0)
 		return (EINVAL);
 	if (ceiling > RTP_PRIO_MAX)
@@ -2064,10 +2101,18 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
 		umtxq_busy(&uq->uq_key);
 		umtxq_unlock(&uq->uq_key);
 
-		save_ceiling = fuword32(&m->m_ceilings[0]);
+		rv = fueword32(&m->m_ceilings[0], &save_ceiling);
+		if (rv == -1) {
+			error = EFAULT;
+			break;
+		}
 
-		owner = casuword32(&m->m_owner,
-		    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+		rv = casueword32(&m->m_owner,
+		    UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED);
+		if (rv == -1) {
+			error = EFAULT;
+			break;
+		}
 
 		if (owner == UMUTEX_CONTESTED) {
 			suword32(&m->m_ceilings[0], ceiling);
@@ -2077,12 +2122,6 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
 			break;
 		}
 
-		/* The address was invalid. */
-		if (owner == -1) {
-			error = EFAULT;
-			break;
-		}
-
 		if ((owner & ~UMUTEX_CONTESTED) == id) {
 			suword32(&m->m_ceilings[0], ceiling);
 			error = 0;
@@ -2129,8 +2168,8 @@ do_lock_umutex(struct thread *td, struct umutex *m,
 	uint32_t flags;
 	int error;
 
-	flags = fuword32(&m->m_flags);
-	if (flags == -1)
+	error = fueword32(&m->m_flags, &flags);
+	if (error == -1)
 		return (EFAULT);
 
 	switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
@@ -2164,9 +2203,10 @@ static int
 do_unlock_umutex(struct thread *td, struct umutex *m)
 {
 	uint32_t flags;
+	int error;
 
-	flags = fuword32(&m->m_flags);
-	if (flags == -1)
+	error = fueword32(&m->m_flags, &flags);
+	if (error == -1)
 		return (EFAULT);
 
 	switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
@@ -2187,21 +2227,27 @@ do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
 {
 	struct abs_timeout timo;
 	struct umtx_q *uq;
-	uint32_t flags;
-	uint32_t clockid;
+	uint32_t flags, clockid, hasw;
 	int error;
 
 	uq = td->td_umtxq;
-	flags = fuword32(&cv->c_flags);
+	error = fueword32(&cv->c_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
 	if (error != 0)
 		return (error);
 
 	if ((wflags & CVWAIT_CLOCKID) != 0) {
-		clockid = fuword32(&cv->c_clockid);
+		error = fueword32(&cv->c_clockid, &clockid);
+		if (error == -1) {
+			umtx_key_release(&uq->uq_key);
+			return (EFAULT);
+		}
 		if (clockid < CLOCK_REALTIME ||
 		    clockid >= CLOCK_THREAD_CPUTIME_ID) {
 			/* hmm, only HW clock id will work. */
+			umtx_key_release(&uq->uq_key);
 			return (EINVAL);
 		}
 	} else {
@@ -2217,12 +2263,12 @@ do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m,
 	 * Set c_has_waiters to 1 before releasing user mutex, also
 	 * don't modify cache line when unnecessary.
 	 */
-	if (fuword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters)) == 0)
+	error = fueword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters),
+	    &hasw);
+	if (error == 0 && hasw == 0)
 		suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1);
 
-	umtxq_lock(&uq->uq_key);
-	umtxq_unbusy(&uq->uq_key);
-	umtxq_unlock(&uq->uq_key);
+	umtxq_unbusy_unlocked(&uq->uq_key);
 
 	error = do_unlock_umutex(td, m);
 
@@ -2276,7 +2322,9 @@ do_cv_signal(struct thread *td, struct ucond *cv)
 	int error, cnt, nwake;
 	uint32_t flags;
 
-	flags = fuword32(&cv->c_flags);
+	error = fueword32(&cv->c_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
 		return (error);	
 	umtxq_lock(&key);
@@ -2287,6 +2335,8 @@ do_cv_signal(struct thread *td, struct ucond *cv)
 		umtxq_unlock(&key);
 		error = suword32(
 		    __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
+		if (error == -1)
+			error = EFAULT;
 		umtxq_lock(&key);
 	}
 	umtxq_unbusy(&key);
@@ -2302,7 +2352,9 @@ do_cv_broadcast(struct thread *td, struct ucond *cv)
 	int error;
 	uint32_t flags;
 
-	flags = fuword32(&cv->c_flags);
+	error = fueword32(&cv->c_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0)
 		return (error);	
 
@@ -2312,10 +2364,10 @@ do_cv_broadcast(struct thread *td, struct ucond *cv)
 	umtxq_unlock(&key);
 
 	error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0);
+	if (error == -1)
+		error = EFAULT;
 
-	umtxq_lock(&key);
-	umtxq_unbusy(&key);
-	umtxq_unlock(&key);
+	umtxq_unbusy_unlocked(&key);
 
 	umtx_key_release(&key);
 	return (error);
@@ -2329,10 +2381,12 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx
 	uint32_t flags, wrflags;
 	int32_t state, oldstate;
 	int32_t blocked_readers;
-	int error;
+	int error, rv;
 
 	uq = td->td_umtxq;
-	flags = fuword32(&rwlock->rw_flags);
+	error = fueword32(&rwlock->rw_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
 	if (error != 0)
 		return (error);
@@ -2345,15 +2399,22 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx
 		wrflags |= URWLOCK_WRITE_WAITERS;
 
 	for (;;) {
-		state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+		rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+		    &state);
+		if (rv == -1) {
+			umtx_key_release(&uq->uq_key);
+			return (EFAULT);
+		}
+
 		/* try to lock it */
 		while (!(state & wrflags)) {
 			if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) {
 				umtx_key_release(&uq->uq_key);
 				return (EAGAIN);
 			}
-			oldstate = casuword32(&rwlock->rw_state, state, state + 1);
-			if (oldstate == -1) {
+			rv = casueword32(&rwlock->rw_state, state,
+			    &oldstate, state + 1);
+			if (rv == -1) {
 				umtx_key_release(&uq->uq_key);
 				return (EFAULT);
 			}
@@ -2379,12 +2440,17 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx
 		 * re-read the state, in case it changed between the try-lock above
 		 * and the check below
 		 */
-		state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+		rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+		    &state);
+		if (rv == -1)
+			error = EFAULT;
 
 		/* set read contention bit */
-		while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) {
-			oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS);
-			if (oldstate == -1) {
+		while (error == 0 && (state & wrflags) &&
+		    !(state & URWLOCK_READ_WAITERS)) {
+			rv = casueword32(&rwlock->rw_state, state,
+			    &oldstate, state | URWLOCK_READ_WAITERS);
+			if (rv == -1) {
 				error = EFAULT;
 				break;
 			}
@@ -2396,17 +2462,13 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx
 				break;
 		}
 		if (error != 0) {
-			umtxq_lock(&uq->uq_key);
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_unlock(&uq->uq_key);
+			umtxq_unbusy_unlocked(&uq->uq_key);
 			break;
 		}
 
 		/* state is changed while setting flags, restart */
 		if (!(state & wrflags)) {
-			umtxq_lock(&uq->uq_key);
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_unlock(&uq->uq_key);
+			umtxq_unbusy_unlocked(&uq->uq_key);
 			error = umtxq_check_susp(td);
 			if (error != 0)
 				break;
@@ -2415,7 +2477,13 @@ do_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, struct _umtx
 
 sleep:
 		/* contention bit is set, before sleeping, increase read waiter count */
-		blocked_readers = fuword32(&rwlock->rw_blocked_readers);
+		rv = fueword32(&rwlock->rw_blocked_readers,
+		    &blocked_readers);
+		if (rv == -1) {
+			umtxq_unbusy_unlocked(&uq->uq_key);
+			error = EFAULT;
+			break;
+		}
 		suword32(&rwlock->rw_blocked_readers, blocked_readers+1);
 
 		while (state & wrflags) {
@@ -2431,18 +2499,32 @@ sleep:
 			umtxq_unlock(&uq->uq_key);
 			if (error)
 				break;
-			state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+			rv = fueword32(__DEVOLATILE(int32_t *,
+			    &rwlock->rw_state), &state);
+			if (rv == -1) {
+				error = EFAULT;
+				break;
+			}
 		}
 
 		/* decrease read waiter count, and may clear read contention bit */
-		blocked_readers = fuword32(&rwlock->rw_blocked_readers);
+		rv = fueword32(&rwlock->rw_blocked_readers,
+		    &blocked_readers);
+		if (rv == -1) {
+			umtxq_unbusy_unlocked(&uq->uq_key);
+			error = EFAULT;
+			break;
+		}
 		suword32(&rwlock->rw_blocked_readers, blocked_readers-1);
 		if (blocked_readers == 1) {
-			state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
-			for (;;) {
-				oldstate = casuword32(&rwlock->rw_state, state,
-					 state & ~URWLOCK_READ_WAITERS);
-				if (oldstate == -1) {
+			rv = fueword32(__DEVOLATILE(int32_t *,
+			    &rwlock->rw_state), &state);
+			if (rv == -1)
+				error = EFAULT;
+			while (error == 0) {
+				rv = casueword32(&rwlock->rw_state, state,
+				    &oldstate, state & ~URWLOCK_READ_WAITERS);
+				if (rv == -1) {
 					error = EFAULT;
 					break;
 				}
@@ -2450,14 +2532,10 @@ sleep:
 					break;
 				state = oldstate;
 				error = umtxq_check_susp(td);
-				if (error != 0)
-					break;
 			}
 		}
 
-		umtxq_lock(&uq->uq_key);
-		umtxq_unbusy(&uq->uq_key);
-		umtxq_unlock(&uq->uq_key);
+		umtxq_unbusy_unlocked(&uq->uq_key);
 		if (error != 0)
 			break;
 	}
@@ -2476,10 +2554,12 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo
 	int32_t state, oldstate;
 	int32_t blocked_writers;
 	int32_t blocked_readers;
-	int error;
+	int error, rv;
 
 	uq = td->td_umtxq;
-	flags = fuword32(&rwlock->rw_flags);
+	error = fueword32(&rwlock->rw_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
 	if (error != 0)
 		return (error);
@@ -2489,10 +2569,16 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo
 
 	blocked_readers = 0;
 	for (;;) {
-		state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+		rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+		    &state);
+		if (rv == -1) {
+			umtx_key_release(&uq->uq_key);
+			return (EFAULT);
+		}
 		while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
-			oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER);
-			if (oldstate == -1) {
+			rv = casueword32(&rwlock->rw_state, state,
+			    &oldstate, state | URWLOCK_WRITE_OWNER);
+			if (rv == -1) {
 				umtx_key_release(&uq->uq_key);
 				return (EFAULT);
 			}
@@ -2528,12 +2614,17 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo
 		 * re-read the state, in case it changed between the try-lock above
 		 * and the check below
 		 */
-		state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+		rv = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state),
+		    &state);
+		if (rv == -1)
+			error = EFAULT;
 
-		while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) &&
-		       (state & URWLOCK_WRITE_WAITERS) == 0) {
-			oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS);
-			if (oldstate == -1) {
+		while (error == 0 && ((state & URWLOCK_WRITE_OWNER) ||
+		    URWLOCK_READER_COUNT(state) != 0) &&
+		    (state & URWLOCK_WRITE_WAITERS) == 0) {
+			rv = casueword32(&rwlock->rw_state, state,
+			    &oldstate, state | URWLOCK_WRITE_WAITERS);
+			if (rv == -1) {
 				error = EFAULT;
 				break;
 			}
@@ -2545,23 +2636,25 @@ do_rw_wrlock(struct thread *td, struct urwlock *rwlock, struct _umtx_time *timeo
 				break;
 		}
 		if (error != 0) {
-			umtxq_lock(&uq->uq_key);
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_unlock(&uq->uq_key);
+			umtxq_unbusy_unlocked(&uq->uq_key);
 			break;
 		}
 
 		if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) {
-			umtxq_lock(&uq->uq_key);
-			umtxq_unbusy(&uq->uq_key);
-			umtxq_unlock(&uq->uq_key);
+			umtxq_unbusy_unlocked(&uq->uq_key);
 			error = umtxq_check_susp(td);
 			if (error != 0)
 				break;
 			continue;
 		}
 sleep:
-		blocked_writers = fuword32(&rwlock->rw_blocked_writers);
+		rv = fueword32(&rwlock->rw_blocked_writers,
+		    &blocked_writers);
+		if (rv == -1) {
+			umtxq_unbusy_unlocked(&uq->uq_key);
+			error = EFAULT;
+			break;
+		}
 		suword32(&rwlock->rw_blocked_writers, blocked_writers+1);
 
 		while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) {
@@ -2577,17 +2670,34 @@ sleep:
 			umtxq_unlock(&uq->uq_key);
 			if (error)
 				break;
-			state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+			rv = fueword32(__DEVOLATILE(int32_t *,
+			    &rwlock->rw_state), &state);
+			if (rv == -1) {
+				error = EFAULT;
+				break;
+			}
 		}
 
-		blocked_writers = fuword32(&rwlock->rw_blocked_writers);
+		rv = fueword32(&rwlock->rw_blocked_writers,
+		    &blocked_writers);
+		if (rv == -1) {
+			umtxq_unbusy_unlocked(&uq->uq_key);
+			error = EFAULT;
+			break;
+		}
 		suword32(&rwlock->rw_blocked_writers, blocked_writers-1);
 		if (blocked_writers == 1) {
-			state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+			rv = fueword32(__DEVOLATILE(int32_t *,
+			    &rwlock->rw_state), &state);
+			if (rv == -1) {
+				umtxq_unbusy_unlocked(&uq->uq_key);
+				error = EFAULT;
+				break;
+			}
 			for (;;) {
-				oldstate = casuword32(&rwlock->rw_state, state,
-					 state & ~URWLOCK_WRITE_WAITERS);
-				if (oldstate == -1) {
+				rv = casueword32(&rwlock->rw_state, state,
+				    &oldstate, state & ~URWLOCK_WRITE_WAITERS);
+				if (rv == -1) {
 					error = EFAULT;
 					break;
 				}
@@ -2603,13 +2713,17 @@ sleep:
 				if (error != 0)
 					break;
 			}
-			blocked_readers = fuword32(&rwlock->rw_blocked_readers);
+			rv = fueword32(&rwlock->rw_blocked_readers,
+			    &blocked_readers);
+			if (rv == -1) {
+				umtxq_unbusy_unlocked(&uq->uq_key);
+				error = EFAULT;
+				break;
+			}
 		} else
 			blocked_readers = 0;
 
-		umtxq_lock(&uq->uq_key);
-		umtxq_unbusy(&uq->uq_key);
-		umtxq_unlock(&uq->uq_key);
+		umtxq_unbusy_unlocked(&uq->uq_key);
 	}
 
 	umtx_key_release(&uq->uq_key);
@@ -2624,20 +2738,26 @@ do_rw_unlock(struct thread *td, struct urwlock *rwlock)
 	struct umtx_q *uq;
 	uint32_t flags;
 	int32_t state, oldstate;
-	int error, q, count;
+	int error, rv, q, count;
 
 	uq = td->td_umtxq;
-	flags = fuword32(&rwlock->rw_flags);
+	error = fueword32(&rwlock->rw_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
 	if (error != 0)
 		return (error);
 
-	state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state));
+	error = fueword32(__DEVOLATILE(int32_t *, &rwlock->rw_state), &state);
+	if (error == -1) {
+		error = EFAULT;
+		goto out;
+	}
 	if (state & URWLOCK_WRITE_OWNER) {
 		for (;;) {
-			oldstate = casuword32(&rwlock->rw_state, state, 
-				state & ~URWLOCK_WRITE_OWNER);
-			if (oldstate == -1) {
+			rv = casueword32(&rwlock->rw_state, state, 
+			    &oldstate, state & ~URWLOCK_WRITE_OWNER);
+			if (rv == -1) {
 				error = EFAULT;
 				goto out;
 			}
@@ -2655,9 +2775,9 @@ do_rw_unlock(struct thread *td, struct urwlock *rwlock)
 		}
 	} else if (URWLOCK_READER_COUNT(state) != 0) {
 		for (;;) {
-			oldstate = casuword32(&rwlock->rw_state, state,
-				state - 1);
-			if (oldstate == -1) {
+			rv = casueword32(&rwlock->rw_state, state,
+			    &oldstate, state - 1);
+			if (rv == -1) {
 				error = EFAULT;
 				goto out;
 			}
@@ -2716,11 +2836,13 @@ do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
 {
 	struct abs_timeout timo;
 	struct umtx_q *uq;
-	uint32_t flags, count;
-	int error;
+	uint32_t flags, count, count1;
+	int error, rv;
 
 	uq = td->td_umtxq;
-	flags = fuword32(&sem->_flags);
+	error = fueword32(&sem->_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
 	if (error != 0)
 		return (error);
@@ -2732,15 +2854,16 @@ do_sem_wait(struct thread *td, struct _usem *sem, struct _umtx_time *timeout)
 	umtxq_busy(&uq->uq_key);
 	umtxq_insert(uq);
 	umtxq_unlock(&uq->uq_key);
-	casuword32(&sem->_has_waiters, 0, 1);
-	count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count));
-	if (count != 0) {
+	rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
+	if (rv == 0)
+		rv = fueword32(__DEVOLATILE(uint32_t *, &sem->_count), &count);
+	if (rv == -1 || count != 0) {
 		umtxq_lock(&uq->uq_key);
 		umtxq_unbusy(&uq->uq_key);
 		umtxq_remove(uq);
 		umtxq_unlock(&uq->uq_key);
 		umtx_key_release(&uq->uq_key);
-		return (0);
+		return (rv == -1 ? EFAULT : 0);
 	}
 	umtxq_lock(&uq->uq_key);
 	umtxq_unbusy(&uq->uq_key);
@@ -2771,7 +2894,9 @@ do_sem_wake(struct thread *td, struct _usem *sem)
 	int error, cnt;
 	uint32_t flags;
 
-	flags = fuword32(&sem->_flags);
+	error = fueword32(&sem->_flags, &flags);
+	if (error == -1)
+		return (EFAULT);
 	if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
 		return (error);	
 	umtxq_lock(&key);
@@ -2789,6 +2914,8 @@ do_sem_wake(struct thread *td, struct _usem *sem)
 			error = suword32(
 			    __DEVOLATILE(uint32_t *, &sem->_has_waiters), 0);
 			umtxq_lock(&key);
+			if (error == -1)
+				error = EFAULT;
 		}
 	}
 	umtxq_unbusy(&key);
@@ -2804,7 +2931,7 @@ do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
 	struct abs_timeout timo;
 	struct umtx_q *uq;
 	uint32_t count, flags;
-	int error;
+	int error, rv;
 
 	uq = td->td_umtxq;
 	flags = fuword32(&sem->_flags);
@@ -2819,8 +2946,8 @@ do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
 	umtxq_busy(&uq->uq_key);
 	umtxq_insert(uq);
 	umtxq_unlock(&uq->uq_key);
-	count = fuword32(__DEVOLATILE(uint32_t *, &sem->_count));
-	if (count == -1) {
+	rv = fueword32(__DEVOLATILE(uint32_t *, &sem->_count), &count);
+	if (rv == -1) {
 		umtxq_lock(&uq->uq_key);
 		umtxq_unbusy(&uq->uq_key);
 		umtxq_remove(uq);
@@ -2839,8 +2966,8 @@ do_sem2_wait(struct thread *td, struct _usem2 *sem, struct _umtx_time *timeout)
 		}
 		if (count == USEM_HAS_WAITERS)
 			break;
-		count = casuword32(&sem->_count, 0, USEM_HAS_WAITERS);
-		if (count == -1) {
+		rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
+		if (rv == -1) {
 			umtxq_lock(&uq->uq_key);
 			umtxq_unbusy(&uq->uq_key);
 			umtxq_remove(uq);
@@ -2877,10 +3004,12 @@ static int
 do_sem2_wake(struct thread *td, struct _usem2 *sem)
 {
 	struct umtx_key key;
-	int error, cnt;
+	int error, cnt, rv;
 	uint32_t count, flags;
 
-	flags = fuword32(&sem->_flags);
+	rv = fueword32(&sem->_flags, &flags);
+	if (rv == -1)
+		return (EFAULT);
 	if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0)
 		return (error);	
 	umtxq_lock(&key);
@@ -2895,12 +3024,12 @@ do_sem2_wake(struct thread *td, struct _usem2 *sem)
 		 */
 		if (cnt == 1) {
 			umtxq_unlock(&key);
-			count = fuword32(__DEVOLATILE(uint32_t *,
-			    &sem->_count));
-			while (count != -1 && count & USEM_HAS_WAITERS)
-				count = casuword32(&sem->_count, count,
+			rv = fueword32(__DEVOLATILE(uint32_t *, &sem->_count),
+			    &count);
+			while (rv != -1 && count & USEM_HAS_WAITERS)
+				rv = casueword32(&sem->_count, count, &count,
 				    count & ~USEM_HAS_WAITERS);
-			if (count == -1)
+			if (rv == -1)
 				error = EFAULT;
 			umtxq_lock(&key);
 		}