Overhaul the per-CPU support a bit:

- The MI portions of struct globaldata have been consolidated into a MI
  struct pcpu.  The MD per-CPU data are specified via a macro defined in
  machine/pcpu.h.  A macro was chosen over a struct mdpcpu so that the
  interface would be cleaner (PCPU_GET(my_md_field) vs.
  PCPU_GET(md.md_my_md_field)).
- All references to globaldata are changed to pcpu instead.  In a UP kernel,
  this data was stored as global variables which is where the original name
  came from.  In an SMP world this data is per-CPU and ideally private to each
  CPU outside of the context of debuggers.  This also included combining
  machine/globaldata.h and machine/globals.h into machine/pcpu.h.
- The pointer to the thread using the FPU on i386 was renamed from
  npxthread to fpcurthread to be identical with other architectures.
- Make the show pcpu ddb command MI with a MD callout to display MD
  fields.
- The globaldata_register() function was renamed to pcpu_init() and now
  init's MI fields of a struct pcpu in addition to registering it with
  the internal array and list.
- A pcpu_destroy() function was added to remove a struct pcpu from the
  internal array and list.

Tested on:	alpha, i386
Reviewed by:	peter, jake

5 files changed, 91 insertions, 24 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 438f2f9..39ab3a0 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -68,7 +68,6 @@
 #include <sys/conf.h>
 
 #include <machine/cpu.h>
-#include <machine/globals.h>
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c
index b37e27a..92e5cb3 100644
--- a/sys/kern/kern_idle.c
+++ b/sys/kern/kern_idle.c
@@ -37,18 +37,18 @@ static void
 idle_setup(void *dummy)
 {
 #ifdef SMP
-	struct globaldata *gd;
+	struct pcpu *pc;
 #endif
 	struct proc *p;
 	int error;
 
 #ifdef SMP
-	SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
+	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
 		error = kthread_create(idle_proc, NULL, &p,
-		    RFSTOPPED | RFHIGHPID, "idle: cpu%d", gd->gd_cpuid);
-		gd->gd_idlethread = &p->p_thread;
-		if (gd->gd_curthread == NULL)
-			gd->gd_curthread = gd->gd_idlethread;
+		    RFSTOPPED | RFHIGHPID, "idle: cpu%d", pc->pc_cpuid);
+		pc->pc_idlethread = &p->p_thread;
+		if (pc->pc_curthread == NULL)
+			pc->pc_curthread = pc->pc_idlethread;
 #else
 		error = kthread_create(idle_proc, NULL, &p,
 		    RFSTOPPED | RFHIGHPID, "idle");
diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c
index 5c8407c..6badb3d 100644
--- a/sys/kern/kern_ktr.c
+++ b/sys/kern/kern_ktr.c
@@ -42,10 +42,10 @@
 #include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/libkern.h>
+#include <sys/pcpu.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/time.h>
-#include <machine/globals.h>
 #include <machine/stdarg.h>
 
 #include <ddb/ddb.h>
diff --git a/sys/kern/subr_pcpu.c b/sys/kern/subr_pcpu.c
index 10a0d98..8db9e1f 100644
--- a/sys/kern/subr_pcpu.c
+++ b/sys/kern/subr_pcpu.c
@@ -44,32 +44,100 @@
  *    sole CPU as 0.
  */
 
+#include "opt_ddb.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/linker_set.h>
+#include <sys/lock.h>
 #include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <ddb/ddb.h>
 
-static struct globaldata *cpuid_to_globaldata[MAXCPU];
+static struct pcpu *cpuid_to_pcpu[MAXCPU];
 struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead);
 
 /*
- * Register a struct globaldata.
+ * Initialize the MI portions of a struct pcpu.
+ */
+void
+pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
+{
+
+	bzero(pcpu, size);
+	KASSERT(cpuid >= 0 && cpuid < MAXCPU,
+	    ("pcpu_init: invalid cpuid %d", cpuid));
+	pcpu->pc_cpuid = cpuid;
+	cpuid_to_pcpu[cpuid] = pcpu;
+	SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu);
+	cpu_pcpu_init(pcpu, cpuid, size);
+}
+
+/*
+ * Destroy a struct pcpu.
  */
 void
-globaldata_register(struct globaldata *globaldata)
+pcpu_destroy(struct pcpu *pcpu)
 {
 
-	KASSERT(globaldata->gd_cpuid >= 0 && globaldata->gd_cpuid < MAXCPU,
-	    ("globaldata_register: invalid cpuid"));
-	cpuid_to_globaldata[globaldata->gd_cpuid] = globaldata;
-	SLIST_INSERT_HEAD(&cpuhead, globaldata, gd_allcpu);
+	SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu);
+	cpuid_to_pcpu[pcpu->pc_cpuid] = NULL;
 }
 
 /*
- * Locate a struct globaldata by cpu id.
+ * Locate a struct pcpu by cpu id.
  */
-struct globaldata *
-globaldata_find(u_int cpuid)
+struct pcpu *
+pcpu_find(u_int cpuid)
+{
+
+	return (cpuid_to_pcpu[cpuid]);
+}
+
+#ifdef DDB
+DB_SHOW_COMMAND(pcpu, db_show_pcpu)
 {
+	struct pcpu *pc;
+	struct thread *td;
+	int id;
 
-	return (cpuid_to_globaldata[cpuid]);
+	if (have_addr)
+		id = ((addr >> 4) % 16) * 10 + (addr % 16);
+	else
+		id = PCPU_GET(cpuid);
+	pc = pcpu_find(id);
+	if (pc == NULL) {
+		db_printf("CPU %d not found\n", id);
+		return;
+	}
+	db_printf("cpuid        = %d\n", pc->pc_cpuid);
+	db_printf("curthread    = ");
+	td = pc->pc_curthread;
+	if (td != NULL)
+		db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid,
+		    td->td_proc->p_comm);
+	else
+		db_printf("none\n");
+	db_printf("curpcb       = %p\n", pc->pc_curpcb);
+	db_printf("fpcurthread  = ");
+	td = pc->pc_fpcurthread;
+	if (td != NULL)
+		db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid,
+		    td->td_proc->p_comm);
+	else
+		db_printf("none\n");
+	db_printf("idlethread   = ");
+	td = pc->pc_idlethread;
+	if (td != NULL)
+		db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid,
+		    td->td_proc->p_comm);
+	else
+		db_printf("none\n");
+	db_show_mdpcpu(pc);
+		
+#ifdef WITNESS
+	db_printf("spin locks held:\n");
+	witness_list_locks(&pc->pc_spinlocks);
+#endif
 }
+#endif
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 03c6612..6b55133 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -132,7 +132,7 @@ forward_signal(struct thread *td)
 void
 forward_roundrobin(void)
 {
-	struct globaldata *gd;
+	struct pcpu *pc;
 	struct thread *td;
 	u_int id, map;
 
@@ -145,11 +145,11 @@ forward_roundrobin(void)
 	if (!forward_roundrobin_enabled)
 		return;
 	map = 0;
-	SLIST_FOREACH(gd, &cpuhead, gd_allcpu) {
-		td = gd->gd_curthread;
-		id = gd->gd_cpuid;
+	SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
+		td = pc->pc_curthread;
+		id = pc->pc_cpuid;
 		if (id != PCPU_GET(cpuid) && (id & stopped_cpus) == 0 &&
-		    td != gd->gd_idlethread) {
+		    td != pc->pc_idlethread) {
 			td->td_kse->ke_flags |= KEF_NEEDRESCHED;
 			map |= id;
 		}