Compromise for critical*()/cpu_critical*() recommit. Cleanup the interrupt

disablement assumptions in kern_fork.c by adding another API call,
cpu_critical_fork_exit().  Cleanup the td_savecrit field by moving it
from MI to MD.  Temporarily move cpu_critical*() from <arch>/include/cpufunc.h
to <arch>/<arch>/critical.c (stage-2 will clean this up).

Implement interrupt deferral for i386 that allows interrupts to remain
enabled inside critical sections.  This also fixes an IPI interlock bug,
and requires uses of icu_lock to be enclosed in a true interrupt disablement.

This is the stage-1 commit.  Stage-2 will occur after stage-1 has stabilized,
and will move cpu_critical*() into its own header file(s) + other things.
This commit may break non-i386 architectures in trivial ways.  This should
be temporary.

Reviewed by:	core
Approved by:	core

3 files changed, 22 insertions, 18 deletions

diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index d508cbf..723fd62 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -774,12 +774,16 @@ fork_exit(callout, arg, frame)
 
 	td->td_kse->ke_oncpu = PCPU_GET(cpuid);
 	/*
-	 * Setup the sched_lock state so that we can release it.
+	 * Finish setting up thread glue.  We need to initialize
+	 * the thread into a td_critnest=1 state.  Some platforms
+	 * may have already partially or fully initialized td_critnest
+	 * and/or td_md.md_savecrit (when applciable).
+	 *
+	 * see <arch>/<arch>/critical.c
 	 */
 	sched_lock.mtx_lock = (uintptr_t)td;
 	sched_lock.mtx_recurse = 0;
-	td->td_critnest = 1;
-	td->td_savecrit = CRITICAL_FORK;
+	cpu_critical_fork_exit();
 	CTR3(KTR_PROC, "fork_exit: new proc %p (pid %d, %s)", p, p->p_pid,
 	    p->p_comm);
 	if (PCPU_GET(switchtime.sec) == 0)
diff --git a/sys/kern/kern_prot.c b/sys/kern/kern_prot.c
index 3d24bed..6b263e0 100644
--- a/sys/kern/kern_prot.c
+++ b/sys/kern/kern_prot.c
@@ -338,10 +338,9 @@ getgroups(td, uap)
 	ngrp = cred->cr_ngroups;
 	error = copyout((caddr_t)cred->cr_groups, (caddr_t)uap->gidset,
 	    ngrp * sizeof(gid_t));
-	if (error)
-		return (error);
-	td->td_retval[0] = ngrp;
-	return (0);
+	if (error == 0)
+	    td->td_retval[0] = ngrp;
+	return (error);
 }
 
 #ifndef _SYS_SYSPROTO_H_
@@ -1112,11 +1111,9 @@ getresuid(td, uap)
 	struct getresuid_args *uap;
 {
 	struct ucred *cred;
-	struct proc *p = td->td_proc;
 	int error1 = 0, error2 = 0, error3 = 0;
 
-	mtx_lock(&Giant);
-	cred = p->p_ucred;
+	cred = td->td_ucred;
 	if (uap->ruid)
 		error1 = copyout((caddr_t)&cred->cr_ruid,
 		    (caddr_t)uap->ruid, sizeof(cred->cr_ruid));
@@ -1126,7 +1123,6 @@ getresuid(td, uap)
 	if (uap->suid)
 		error3 = copyout((caddr_t)&cred->cr_svuid,
 		    (caddr_t)uap->suid, sizeof(cred->cr_svuid));
-	mtx_unlock(&Giant);
 	return (error1 ? error1 : error2 ? error2 : error3);
 }
 
@@ -1147,11 +1143,9 @@ getresgid(td, uap)
 	struct getresgid_args *uap;
 {
 	struct ucred *cred;
-	struct proc *p = td->td_proc;
 	int error1 = 0, error2 = 0, error3 = 0;
 
-	mtx_lock(&Giant);
-	cred = p->p_ucred;
+	cred = td->td_ucred;
 	if (uap->rgid)
 		error1 = copyout((caddr_t)&cred->cr_rgid,
 		    (caddr_t)uap->rgid, sizeof(cred->cr_rgid));
@@ -1161,7 +1155,6 @@ getresgid(td, uap)
 	if (uap->sgid)
 		error3 = copyout((caddr_t)&cred->cr_svgid,
 		    (caddr_t)uap->sgid, sizeof(cred->cr_svgid));
-	mtx_unlock(&Giant);
 	return (error1 ? error1 : error2 ? error2 : error3);
 }
 
@@ -1233,6 +1226,8 @@ __setugid(td, uap)
 
 /*
  * Check if gid is a member of the group set.
+ *
+ * MPSAFE (cred must be held)
  */
 int
 groupmember(gid, cred)
@@ -1289,6 +1284,8 @@ suser_td(td)
 
 /*
  * wrapper to use if you have the thread on hand but not the proc.
+ *
+ * MPSAFE (cred must be held)
  */
 int
 suser_xxx_td(cred, td, flag)
@@ -1330,6 +1327,8 @@ suser_xxx(cred, proc, flag)
  * existing securelevel checks that occurred without a process/credential
  * context.  In the future this will be disallowed, so a kernel message
  * is displayed.
+ *
+ * MPSAFE
  */
 int
 securelevel_gt(struct ucred *cr, int level)
diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c
index 78505a3..ccfb114 100644
--- a/sys/kern/kern_switch.c
+++ b/sys/kern/kern_switch.c
@@ -77,7 +77,7 @@ critical_enter(void)
 
 	td = curthread;
 	if (td->td_critnest == 0)
-		td->td_savecrit = cpu_critical_enter();
+		cpu_critical_enter();
 	td->td_critnest++;
 }
 
@@ -89,9 +89,10 @@ critical_exit(void)
 	td = curthread;
 	if (td->td_critnest == 1) {
 		td->td_critnest = 0;
-		cpu_critical_exit(td->td_savecrit);
-	} else
+		cpu_critical_exit();
+	} else {
 		td->td_critnest--;
+	}
 }
 
 /*