Add some DIAGNOSTIC code.

While in userland, keep the thread's ucred reference in a shadow
field so that the usual place to store it is NULL.
If DIAGNOSTIC is not set, the thread ucred is kept valid until the next
kernel entry, at which time it is checked against the process cred
and possibly corrected. Produces a BIG speedup in
kernels with INVARIANTS set. (A previous commit corrected it
for the non INVARIANTS case already)

Reviewed by:	dillon@freebsd.org

10 files changed, 190 insertions, 94 deletions

diff --git a/sys/alpha/alpha/trap.c b/sys/alpha/alpha/trap.c
index 2cdf540d..c0b1327 100644
--- a/sys/alpha/alpha/trap.c
+++ b/sys/alpha/alpha/trap.c
@@ -297,7 +297,12 @@ trap(a0, a1, a2, entry, framep)
 	if (user)  {
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = framep;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap: thread got a cred while in userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred)
 			cred_update_thread(td);
 	} else {
@@ -625,12 +630,12 @@ out:
 		framep->tf_regs[FRAME_SP] = alpha_pal_rdusp();
 		userret(td, framep, sticks);
 		mtx_assert(&Giant, MA_NOTOWNED);
-#ifdef	INVARIANTS
-		mtx_lock(&Giant);
-		crfree(td->td_ucred);
-		mtx_unlock(&Giant);
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred_cache)
+			panic("trap: thread already has cached ucred");
+		td->td_ucred_cache = td->td_ucred;
 		td->td_ucred = NULL;
-#endif
+#endif /* DIAGNOSTIC */
 	}
 	return;
 
@@ -699,7 +704,12 @@ syscall(code, framep)
 	td->td_frame = framep;
 	opc = framep->tf_regs[FRAME_PC] - 4;
 	sticks = td->td_kse->ke_sticks;
-	KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred)
+		panic("syscall:thread got a cred while in userspace");
+	td->td_ucred = td->td_ucred_cache;
+	td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 	if (td->td_ucred != p->p_ucred)
 		cred_update_thread(td);
 
@@ -822,12 +832,12 @@ syscall(code, framep)
 	 */
 	STOPEVENT(p, S_SCX, code);
 
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("syscall:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
 	td->td_ucred = NULL;
-#endif
+#endif /* DIAGNOSTIC */
 #ifdef WITNESS
 	if (witness_list(td)) {
 		panic("system call %s returning with mutex(s) held\n",
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index bc31c8c..504fdde 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -255,7 +255,12 @@ trap(frame)
 
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = &frame;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap:thread got a cred while userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred) 
 			cred_update_thread(td);
 
@@ -643,12 +648,12 @@ user:
 	userret(td, &frame, sticks);
 	mtx_assert(&Giant, MA_NOTOWNED);
 userout:
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("trap:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+        td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 out:
 	return;
 }
@@ -954,7 +959,12 @@ syscall(frame)
 
 	sticks = td->td_kse->ke_sticks;
 	td->td_frame = &frame;
-	KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred)
+		panic("trap:thread got a cred while userspace");
+	td->td_ucred = td->td_ucred_cache;
+	td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 	if (td->td_ucred != p->p_ucred) 
 		cred_update_thread(td);
 	params = (caddr_t)frame.tf_esp + sizeof(int);
@@ -1099,12 +1109,13 @@ bad:
 	 */
 	STOPEVENT(p, S_SCX, code);
 
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("syscall:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+        td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
+
 #ifdef WITNESS
 	if (witness_list(td)) {
 		panic("system call %s returning with mutex(s) held\n",
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index bc31c8c..504fdde 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -255,7 +255,12 @@ trap(frame)
 
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = &frame;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap:thread got a cred while userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred) 
 			cred_update_thread(td);
 
@@ -643,12 +648,12 @@ user:
 	userret(td, &frame, sticks);
 	mtx_assert(&Giant, MA_NOTOWNED);
 userout:
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("trap:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+        td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 out:
 	return;
 }
@@ -954,7 +959,12 @@ syscall(frame)
 
 	sticks = td->td_kse->ke_sticks;
 	td->td_frame = &frame;
-	KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred)
+		panic("trap:thread got a cred while userspace");
+	td->td_ucred = td->td_ucred_cache;
+	td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 	if (td->td_ucred != p->p_ucred) 
 		cred_update_thread(td);
 	params = (caddr_t)frame.tf_esp + sizeof(int);
@@ -1099,12 +1109,13 @@ bad:
 	 */
 	STOPEVENT(p, S_SCX, code);
 
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("syscall:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+        td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
+
 #ifdef WITNESS
 	if (witness_list(td)) {
 		panic("system call %s returning with mutex(s) held\n",
diff --git a/sys/ia64/ia64/trap.c b/sys/ia64/ia64/trap.c
index a0ca070..141ae9b 100644
--- a/sys/ia64/ia64/trap.c
+++ b/sys/ia64/ia64/trap.c
@@ -323,7 +323,12 @@ trap(int vector, int imm, struct trapframe *framep)
 	if (user) {
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = framep;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap:thread got a cred while userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred)
 			cred_update_thread(td);
 	} else {
@@ -708,12 +713,12 @@ out:
 	if (user) {
 		userret(td, framep, sticks);
 		mtx_assert(&Giant, MA_NOTOWNED);
-#ifdef	INVARIANTS
-		mtx_lock(&Giant);
-		crfree(td->td_ucred);
-		mtx_unlock(&Giant);
-		td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred_cache)
+			panic("trap:thread already has cached ucred");
+		td->td_ucred_cache = td->td_ucred;
+        	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 	}
 	return;
 
@@ -758,7 +763,12 @@ syscall(int code, u_int64_t *args, struct trapframe *framep)
 
 	td->td_frame = framep;
 	sticks = td->td_kse->ke_sticks;
-	KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred)
+		panic("trap:thread got a cred while userspace");
+	td->td_ucred = td->td_ucred_cache;
+	td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 	if (td->td_ucred != p->p_ucred)
 		cred_update_thread(td);
 
@@ -865,12 +875,12 @@ syscall(int code, u_int64_t *args, struct trapframe *framep)
 	 */
 	STOPEVENT(p, S_SCX, code);
 
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("trap:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+       	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 #ifdef WITNESS
 	if (witness_list(td)) {
 		panic("system call %s returning with mutex(s) held\n",
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 3cd3017..8fe1006 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -477,6 +477,9 @@ again:
 	PROC_LOCK(p1);
 	p2->p_ucred = crhold(p1->p_ucred);
 	td2->td_ucred = crhold(p2->p_ucred);	/* XXXKSE */
+#ifdef	DIAGNOSTIC 			/* see the comment in ast() */
+	td2->td_ucred_cache = NULL;
+#endif
 
 	if (p2->p_args)
 		p2->p_args->ar_ref++;
@@ -802,12 +805,12 @@ fork_exit(callout, arg, frame)
 		kthread_exit(0);
 	}
 	PROC_UNLOCK(p);
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("fork_exit:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+       	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 	mtx_assert(&Giant, MA_NOTOWNED);
 }
 
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index d599b91..cefce4e 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -131,7 +131,6 @@ ast(framep)
 #endif
 
 	KASSERT(TRAPF_USERMODE(framep), ("ast in kernel mode"));
-	KASSERT(td->td_ucred == NULL, ("leaked ucred"));
 #ifdef WITNESS
 	if (witness_list(td))
 		panic("Returning to user mode with mutex(s) held");
@@ -161,6 +160,30 @@ ast(framep)
 			p->p_stats->p_prof.pr_ticks = 0;
 		}
 		mtx_unlock_spin(&sched_lock);
+
+#ifdef DIAGNOSTIC
+		/*
+		 * As a diagnostic tool we make sure that td->td_ucred
+		 * is NULL while we are in user space. This is
+		 * because theoreticaly this field is only defined
+		 * while the thread is in the kernel. Making it NULL
+		 * will immediatly trap invalid usage of this field.
+		 * In practice however we keep the reference to the ucred
+		 * because it's almost always going to be the same cred we will
+		 * need at the next syscall, and it can be expensive
+		 * to keep dropping and reacquiring the reference.
+		 * We thus stash it away elsewhere until we return
+		 * to the kernel, where we bring it back. If 
+		 * DIAGNOSTIC is not defined we don't bother with
+		 * making it NULL, and just leave it in place.
+		 * (don't remove this comment without removing the pointers
+		 * to it in sys/proc.h, */*/trap.c, kern/kern_fork.c and here.)
+		 */
+		if (td->td_ucred)
+			panic("ast:thread got a cred before reaching AST");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred) 
 			cred_update_thread(td);
 		if (flags & KEF_OWEUPC && sflag & PS_PROFIL)
@@ -187,12 +210,13 @@ ast(framep)
 		}
 
 		userret(td, framep, sticks);
-#ifdef	INVARIANTS
-		mtx_lock(&Giant);
-		crfree(td->td_ucred);
-		mtx_unlock(&Giant);
+#ifdef DIAGNOSTIC			/* see comment above */
+		if (td->td_ucred_cache)
+			panic("ast:thread already has cached ucred");
+		td->td_ucred_cache = td->td_ucred;
 		td->td_ucred = NULL;
-#endif
+#endif /* DIAGNOSTIC */
+
 		s = cpu_critical_enter();
 	}
 	mtx_assert(&Giant, MA_NOTOWNED);
diff --git a/sys/powerpc/aim/trap.c b/sys/powerpc/aim/trap.c
index 6526709..5a08c63 100644
--- a/sys/powerpc/aim/trap.c
+++ b/sys/powerpc/aim/trap.c
@@ -226,7 +226,12 @@ trap(struct trapframe *frame)
 	if (user) {
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = frame;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap:thread got a cred while userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred)
 			cred_update_thread(td);
 
@@ -288,6 +293,7 @@ trap(struct trapframe *frame)
 		default:
 			trap_fatal(frame);
 		}
+		/* NOTREACHED */
 	}
 	if (sig != 0) {
 		if (p->p_sysent->sv_transtrap != NULL)
@@ -296,12 +302,12 @@ trap(struct trapframe *frame)
 	}
 	userret(td, frame, sticks);
 	mtx_assert(&Giant, MA_NOTOWNED);
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("trap:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+       	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 }
 
 void
diff --git a/sys/powerpc/powerpc/trap.c b/sys/powerpc/powerpc/trap.c
index 6526709..5a08c63 100644
--- a/sys/powerpc/powerpc/trap.c
+++ b/sys/powerpc/powerpc/trap.c
@@ -226,7 +226,12 @@ trap(struct trapframe *frame)
 	if (user) {
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = frame;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap:thread got a cred while userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred)
 			cred_update_thread(td);
 
@@ -288,6 +293,7 @@ trap(struct trapframe *frame)
 		default:
 			trap_fatal(frame);
 		}
+		/* NOTREACHED */
 	}
 	if (sig != 0) {
 		if (p->p_sysent->sv_transtrap != NULL)
@@ -296,12 +302,12 @@ trap(struct trapframe *frame)
 	}
 	userret(td, frame, sticks);
 	mtx_assert(&Giant, MA_NOTOWNED);
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("trap:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+       	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 }
 
 void
diff --git a/sys/sparc64/sparc64/trap.c b/sys/sparc64/sparc64/trap.c
index b6f47bb..249dd29 100644
--- a/sys/sparc64/sparc64/trap.c
+++ b/sys/sparc64/sparc64/trap.c
@@ -175,7 +175,12 @@ trap(struct trapframe *tf)
 	if ((type & T_KERNEL) == 0) {
 		sticks = td->td_kse->ke_sticks;
 		td->td_frame = tf;
-		KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+		if (td->td_ucred)
+			panic("trap:thread got a cred while userspace");
+		td->td_ucred = td->td_ucred_cache;
+		td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 		if (td->td_ucred != p->p_ucred)
 			cred_update_thread(td);
  	} else {
@@ -379,12 +384,12 @@ trapsig:
 user:
 	userret(td, tf, sticks);
 	mtx_assert(&Giant, MA_NOTOWNED);
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("trap:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+       	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 out:
 	CTR1(KTR_TRAP, "trap: td=%p return", td);
 	return;
@@ -540,7 +545,12 @@ syscall(struct trapframe *tf)
 
 	sticks = td->td_kse->ke_sticks;
 	td->td_frame = tf;
-	KASSERT(td->td_ucred == NULL, ("already have a ucred"));
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred)
+		panic("syscall:thread got a cred while userspace");
+	td->td_ucred = td->td_ucred_cache;
+	td->td_ucred_cache = NULL;
+#endif /* DIAGNOSTIC */
 	if (td->td_ucred != p->p_ucred)
 		cred_update_thread(td);
 	code = tf->tf_global[1];
@@ -677,12 +687,12 @@ bad:
 	 */
 	STOPEVENT(p, S_SCX, code);
 
-#ifdef	INVARIANTS
-	mtx_lock(&Giant);
-	crfree(td->td_ucred);
-	mtx_unlock(&Giant);
-	td->td_ucred = NULL;
-#endif
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	if (td->td_ucred_cache)
+		panic("syscall:thread already has cached ucred");
+	td->td_ucred_cache = td->td_ucred;
+       	td->td_ucred = NULL;
+#endif /* DIAGNOSTIC */
 #ifdef WITNESS
 	if (witness_list(td)) {
 		panic("system call %s returning with mutex(s) held\n",
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 6a7d20d..fd18f1b 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -272,6 +272,11 @@ struct thread {
 #define	td_endcopy td_pcb
 
 	struct ucred	*td_ucred;	/* (k) Reference to credentials. */
+#ifdef DIAGNOSTIC 			/* see the comment in ast() */
+	struct ucred	*td_ucred_cache; /* (k) hide cred here for DIAGNOSTIC */
+#else
+	void 		*td_dontuse;	/* keep the size the same if not DIAG */
+#endif
 	struct pcb	*td_pcb;	/* (k) Kernel VA of pcb and kstack. */
 	struct callout	td_slpcallout;	/* (h) Callout for sleep. */
 	struct trapframe *td_frame;	/* (k) */