Update comments about _start, the kernel entry point, to reflect new

parameters needed for smp support.
If we are not the boot processor, jump to the smp startup code instead.
Implement a per-cpu panic stack, which is used for bootstrapping both
primary and secondary processors and during faults on the kernel stack.
Arrange the per-cpu page like the pcb, with the struct pcpu at the end
of the page and the panic stack before it.
Use the boot processor's panic stack for calling sparc64_init.
Split the code to set preloaded global registers and to map the kernel
tsb out into functions, which non-boot processors can call.
Allocate the kstack for thread0 dynamically in pmap_bootstrap, and give
it a guard page too.

4 files changed, 321 insertions, 101 deletions

diff --git a/sys/sparc64/sparc64/locore.S b/sys/sparc64/sparc64/locore.S
index 18e6e54..5d7e5d3 100644
--- a/sys/sparc64/sparc64/locore.S
+++ b/sys/sparc64/sparc64/locore.S
@@ -28,8 +28,10 @@
 
 #include <sys/syscall.h>
 
+#include <machine/asi.h>
 #include <machine/asmacros.h>
 #include <machine/pstate.h>
+#include <machine/upa.h>
 
 #include "assym.s"
 
@@ -39,31 +41,145 @@
 	.set	kernbase,KERNBASE
 
 /*
- * void _start(caddr_t metadata, u_long o1, u_long o2, u_long o3, 
+ * void _start(caddr_t metadata, u_int *state, u_int mid, u_int bootmid,
  *	       u_long ofw_vec)
+ *
+ * XXX: in am smp system the other cpus are started in the loader, but since
+ * there's no way to look up a symbol there, we need to use the same entry
+ * point.  So if the module id is not equal to bootcpu, jump to _mp_start.
  */
 ENTRY(_start)
-	wrpr	%g0, PSTATE_IE | PSTATE_PRIV | PSTATE_PEF, %pstate
-	mov	%o0, %g1
-	mov	%o4, %g2
+	/*
+	 * Initialize misc state to known values.  Interrupts disabled, normal
+	 * globals, windows flushed (cr = 0, cs = nwindows - 1), no clean
+	 * windows, pil 0, and floating point disabled.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
 	flushw
-	wrpr	%g0, 1, %cwp
 	wrpr	%g0, 0, %cleanwin
 	wrpr	%g0, 0, %pil
 	wr	%g0, 0, %fprs
 
-	SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l0, %o0)
-	sub	%o0, SPOFF + CCFSZ, %sp
+#ifdef SMP
+	/*
+	 * If we're not the boot processor, go do other stuff.
+	 */
+	cmp	%o2, %o3
+	be	%xcc, 1f
+	 nop
+	call	_mp_start
+	 nop
+	sir
+1:
+#endif
+
+	/*
+	 * Get onto our per-cpu panic stack, which precedes the struct pcpu in
+	 * the per-cpu page.
+	 */
+	SET(pcpu0 + PAGE_SIZE - PC_SIZEOF, %l1, %l0)
+	sub	%l0, SPOFF + CCFSZ, %sp
+
+	/*
+	 * Enable interrupts.
+	 */
+	wrpr	%g0, PSTATE_KERNEL, %pstate
 
-	mov	%g1, %o0
+	/*
+	 * Do initial bootstrap to setup pmap and thread0.
+	 */
 	call	sparc64_init
-	 mov	%g2, %o1
+	 nop
+
+	/*
+	 * Get onto thread0's kstack.
+	 */
+	sub	PCB_REG, SPOFF + CCFSZ, %sp
+
+	/*
+	 * And away we go.  This doesn't return.
+	 */
 	call	mi_startup
 	 nop
+	sir
 	! NOTREACHED
 END(_start)
 
 /*
+ * void cpu_setregs(struct pcpu *pc)
+ */
+ENTRY(cpu_setregs)
+	ldx	[%o0 + PC_CURTHREAD], %o1
+	ldx	[%o1 + TD_PCB], %o1
+
+	/*
+	 * Disable interrupts, normal globals.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	/*
+	 * Normal %g6 points to the current thread's pcb, and %g7 points to
+	 * the per-cpu data structure.
+	 */
+	mov	%o1, PCB_REG
+	mov	%o0, PCPU_REG
+
+	/*
+	 * Alternate globals.
+	 */
+	wrpr	%g0, PSTATE_ALT, %pstate
+
+	/*
+	 * Alternate %g5 points to a per-cpu panic stack, %g6 points to the
+	 * current thread's pcb, and %g7 points to the per-cpu data structure.
+	 */
+	mov	%o0, ASP_REG
+	mov	%o1, PCB_REG
+	mov	%o0, PCPU_REG
+
+	/*
+	 * Interrupt globals.
+	 */
+	wrpr	%g0, PSTATE_INTR, %pstate
+
+	/*
+	 * Interrupt %g7 points to the per-cpu data structure.
+	 */
+	mov	%o0, PCPU_REG
+
+	/*
+	 * MMU globals.
+	 */
+	wrpr	%g0, PSTATE_MMU, %pstate
+
+	/*
+	 * MMU %g7 points to the user tsb.  Initialize it to something sane
+	 * here to catch invalid use.
+	 */
+	mov	%g0, TSB_REG
+
+	/*
+	 * Normal globals again.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	/*
+	 * Force trap level 1 and take over the trap table.
+	 */
+	SET(tl0_base, %o2, %o1)
+	wrpr	%g0, 1, %tl
+	wrpr	%o1, 0, %tba
+
+	/*
+	 * Re-enable interrupts.
+	 */
+	wrpr	%g0, PSTATE_KERNEL, %pstate
+
+	retl
+	 nop
+END(cpu_setregs)
+
+/*
  * Signal trampoline, copied out to user stack.  Must be 16 byte aligned or
  * the argv and envp pointers can become misaligned.
  */
diff --git a/sys/sparc64/sparc64/locore.s b/sys/sparc64/sparc64/locore.s
index 18e6e54..5d7e5d3 100644
--- a/sys/sparc64/sparc64/locore.s
+++ b/sys/sparc64/sparc64/locore.s
@@ -28,8 +28,10 @@
 
 #include <sys/syscall.h>
 
+#include <machine/asi.h>
 #include <machine/asmacros.h>
 #include <machine/pstate.h>
+#include <machine/upa.h>
 
 #include "assym.s"
 
@@ -39,31 +41,145 @@
 	.set	kernbase,KERNBASE
 
 /*
- * void _start(caddr_t metadata, u_long o1, u_long o2, u_long o3, 
+ * void _start(caddr_t metadata, u_int *state, u_int mid, u_int bootmid,
  *	       u_long ofw_vec)
+ *
+ * XXX: in am smp system the other cpus are started in the loader, but since
+ * there's no way to look up a symbol there, we need to use the same entry
+ * point.  So if the module id is not equal to bootcpu, jump to _mp_start.
  */
 ENTRY(_start)
-	wrpr	%g0, PSTATE_IE | PSTATE_PRIV | PSTATE_PEF, %pstate
-	mov	%o0, %g1
-	mov	%o4, %g2
+	/*
+	 * Initialize misc state to known values.  Interrupts disabled, normal
+	 * globals, windows flushed (cr = 0, cs = nwindows - 1), no clean
+	 * windows, pil 0, and floating point disabled.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
 	flushw
-	wrpr	%g0, 1, %cwp
 	wrpr	%g0, 0, %cleanwin
 	wrpr	%g0, 0, %pil
 	wr	%g0, 0, %fprs
 
-	SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l0, %o0)
-	sub	%o0, SPOFF + CCFSZ, %sp
+#ifdef SMP
+	/*
+	 * If we're not the boot processor, go do other stuff.
+	 */
+	cmp	%o2, %o3
+	be	%xcc, 1f
+	 nop
+	call	_mp_start
+	 nop
+	sir
+1:
+#endif
+
+	/*
+	 * Get onto our per-cpu panic stack, which precedes the struct pcpu in
+	 * the per-cpu page.
+	 */
+	SET(pcpu0 + PAGE_SIZE - PC_SIZEOF, %l1, %l0)
+	sub	%l0, SPOFF + CCFSZ, %sp
+
+	/*
+	 * Enable interrupts.
+	 */
+	wrpr	%g0, PSTATE_KERNEL, %pstate
 
-	mov	%g1, %o0
+	/*
+	 * Do initial bootstrap to setup pmap and thread0.
+	 */
 	call	sparc64_init
-	 mov	%g2, %o1
+	 nop
+
+	/*
+	 * Get onto thread0's kstack.
+	 */
+	sub	PCB_REG, SPOFF + CCFSZ, %sp
+
+	/*
+	 * And away we go.  This doesn't return.
+	 */
 	call	mi_startup
 	 nop
+	sir
 	! NOTREACHED
 END(_start)
 
 /*
+ * void cpu_setregs(struct pcpu *pc)
+ */
+ENTRY(cpu_setregs)
+	ldx	[%o0 + PC_CURTHREAD], %o1
+	ldx	[%o1 + TD_PCB], %o1
+
+	/*
+	 * Disable interrupts, normal globals.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	/*
+	 * Normal %g6 points to the current thread's pcb, and %g7 points to
+	 * the per-cpu data structure.
+	 */
+	mov	%o1, PCB_REG
+	mov	%o0, PCPU_REG
+
+	/*
+	 * Alternate globals.
+	 */
+	wrpr	%g0, PSTATE_ALT, %pstate
+
+	/*
+	 * Alternate %g5 points to a per-cpu panic stack, %g6 points to the
+	 * current thread's pcb, and %g7 points to the per-cpu data structure.
+	 */
+	mov	%o0, ASP_REG
+	mov	%o1, PCB_REG
+	mov	%o0, PCPU_REG
+
+	/*
+	 * Interrupt globals.
+	 */
+	wrpr	%g0, PSTATE_INTR, %pstate
+
+	/*
+	 * Interrupt %g7 points to the per-cpu data structure.
+	 */
+	mov	%o0, PCPU_REG
+
+	/*
+	 * MMU globals.
+	 */
+	wrpr	%g0, PSTATE_MMU, %pstate
+
+	/*
+	 * MMU %g7 points to the user tsb.  Initialize it to something sane
+	 * here to catch invalid use.
+	 */
+	mov	%g0, TSB_REG
+
+	/*
+	 * Normal globals again.
+	 */
+	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	/*
+	 * Force trap level 1 and take over the trap table.
+	 */
+	SET(tl0_base, %o2, %o1)
+	wrpr	%g0, 1, %tl
+	wrpr	%o1, 0, %tba
+
+	/*
+	 * Re-enable interrupts.
+	 */
+	wrpr	%g0, PSTATE_KERNEL, %pstate
+
+	retl
+	 nop
+END(cpu_setregs)
+
+/*
  * Signal trampoline, copied out to user stack.  Must be 16 byte aligned or
  * the argv and envp pointers can become misaligned.
  */
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index 0632364..ebeb5d7 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -106,33 +106,20 @@
 
 typedef int ofw_vec_t(void *);
 
-extern char tl0_base[];
-
-extern char _end[];
-
 int physmem;
 int cold = 1;
 long dumplo;
 int Maxmem;
 
-u_long debug_mask;
-
 struct mtx Giant;
 struct mtx sched_lock;
 
-static struct pcpu __pcpu;
-/*
- * This needs not be aligned as the other user areas, provided that process 0
- * does not have an fp state (which it doesn't normally).
- * This constraint is only here for debugging.
- */
-char kstack0[KSTACK_PAGES * PAGE_SIZE] __attribute__((aligned(64)));
-static char uarea0[UAREA_PAGES * PAGE_SIZE] __attribute__((aligned(64)));
-static struct trapframe frame0;
-struct user *proc0uarea;
-vm_offset_t proc0kstack;
+char pcpu0[PAGE_SIZE];
+char uarea0[UAREA_PAGES * PAGE_SIZE];
+struct trapframe frame0;
 
-char panic_stack[PANIC_STACK_PAGES * PAGE_SIZE];
+vm_offset_t kstack0;
+vm_offset_t kstack0_phys;
 
 struct kva_md_info kmi;
 
@@ -142,12 +129,15 @@ u_long ofw_tba;
 static struct timecounter tick_tc;
 
 static timecounter_get_t tick_get_timecount;
-void sparc64_init(caddr_t mdp, ofw_vec_t *vec);
+void sparc64_init(caddr_t mdp, u_int *state, u_int mid, u_int bootmid,
+		  ofw_vec_t *vec);
 void sparc64_shutdown_final(void *dummy, int howto);
 
 static void cpu_startup(void *);
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 
+CTASSERT(sizeof(struct pcpu) <= (PAGE_SIZE / 2));
+
 static void
 cpu_startup(void *arg)
 {
@@ -199,12 +189,13 @@ tick_get_timecount(struct timecounter *tc)
 }
 
 void
-sparc64_init(caddr_t mdp, ofw_vec_t *vec)
+sparc64_init(caddr_t mdp, u_int *state, u_int mid, u_int bootmid,
+	     ofw_vec_t *vec)
 {
+	struct pcpu *pc;
 	vm_offset_t end;
 	vm_offset_t off;
 	caddr_t kmdp;
-	u_long ps;
 
 	end = 0;
 	kmdp = NULL;
@@ -276,20 +267,20 @@ sparc64_init(caddr_t mdp, ofw_vec_t *vec)
 	 */
 	tick_stop();
 
-	/* Initialize the interrupt tables. */
+	/*
+	 * Initialize the interrupt tables.
+	 */
 	intr_init();
 
-	proc_linkup(&proc0);
 	/*
 	 * Initialize proc0 stuff (p_contested needs to be done early).
 	 */
-	proc0uarea = (struct user *)uarea0;
-	proc0kstack = (vm_offset_t)kstack0;
+	proc_linkup(&proc0);
 	proc0.p_md.md_utrap = NULL;
-	proc0.p_uarea = proc0uarea;
+	proc0.p_uarea = (struct user *)uarea0;
 	proc0.p_stats = &proc0.p_uarea->u_stats;
 	thread0 = &proc0.p_thread;
-	thread0->td_kstack = proc0kstack;
+	thread0->td_kstack = kstack0;
 	thread0->td_pcb = (struct pcb *)
 	    (thread0->td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
 	frame0.tf_tstate = TSTATE_IE | TSTATE_PEF;
@@ -297,46 +288,20 @@ sparc64_init(caddr_t mdp, ofw_vec_t *vec)
 	LIST_INIT(&thread0->td_contested);
 
 	/*
-	 * Force trap level 1 and take over the trap table.
+	 * Prime our per-cpu data page for use.  Note, we are using it for our
+	 * stack, so don't pass the real size (PAGE_SIZE) to pcpu_init or
+	 * it'll zero it out from under us.
 	 */
-	ps = rdpr(pstate);
-	wrpr(pstate, ps, PSTATE_IE);
-	wrpr(tl, 0, 1);
-	wrpr(tba, tl0_base, 0);
+	pc = (struct pcpu *)(pcpu0 + PAGE_SIZE) - 1;
+	pcpu_init(pc, 0, sizeof(struct pcpu));
+	pc->pc_curthread = thread0;
+	pc->pc_curpcb = thread0->td_pcb;
+	pc->pc_mid = mid;
 
 	/*
-	 * Setup preloaded globals.
-	 * Normal %g7 points to the per-cpu data structure.
+	 * Initialize global registers.
 	 */
-	__asm __volatile("mov %0, %%g7" : : "r" (&__pcpu));
-
-	/*
-	 * Alternate %g5 points to a per-cpu stack for saving alternate
-	 * globals, %g6 points to the current thread's pcb, and %g7 points
-	 * to the per-cpu data structure.
-	 */
-	wrpr(pstate, ps, PSTATE_AG | PSTATE_IE);
-	__asm __volatile("mov %0, %%g5" : : "r"
-	    (&__pcpu.pc_alt_stack[ALT_STACK_SIZE - 1]));
-	__asm __volatile("mov %0, %%g6" : : "r" (thread0->td_pcb));
-	__asm __volatile("mov %0, %%g7" : : "r" (&__pcpu));
-
-	/*
-	 * Interrupt %g6 points to a per-cpu interrupt queue, %g7 points to
-	 * the interrupt vector table.
-	 */
-	wrpr(pstate, ps, PSTATE_IG | PSTATE_IE);
-	__asm __volatile("mov %0, %%g6" : : "r" (&__pcpu.pc_iq));
-	__asm __volatile("mov %0, %%g7" : : "r" (&intr_vectors));
-
-	/*
-	 * MMU %g7 points to the user tsb.  Initialize it to something sane
-	 * here to catch invalid use.
-	 */
-	wrpr(pstate, ps, PSTATE_MG | PSTATE_IE);
-	__asm __volatile("mov %%g0, %%g7" : :);
-
-	wrpr(pstate, ps, 0);
+	cpu_setregs(pc);
 
 	/*
 	 * Map and initialize the message buffer (after setting trap table).
@@ -346,13 +311,6 @@ sparc64_init(caddr_t mdp, ofw_vec_t *vec)
 	msgbufinit(msgbufp, MSGBUF_SIZE);
 
 	/*
-	 * Initialize curthread so that mutexes work.
-	 */
-	pcpu_init(pcpup, 0, sizeof(struct pcpu));
-	PCPU_SET(curthread, thread0);
-	PCPU_SET(curpcb, thread0->td_pcb);
-
-	/*
 	 * Initialize mutexes.
 	 */
 	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c
index 55c1729..1211b18 100644
--- a/sys/sparc64/sparc64/pmap.c
+++ b/sys/sparc64/sparc64/pmap.c
@@ -308,23 +308,23 @@ pmap_bootstrap(vm_offset_t ekva)
 	tsb_kernel_phys = pa;
 	tsb_kernel = (struct tte *)virtual_avail;
 	virtual_avail += KVA_PAGES * PAGE_SIZE_4M;
-	for (i = 0; i < KVA_PAGES; i++) {
-		va = (vm_offset_t)tsb_kernel + i * PAGE_SIZE_4M;
-		tte.tte_tag = TT_CTX(TLB_CTX_KERNEL) | TT_VA(va);
-		tte.tte_data = TD_V | TD_4M | TD_VA_LOW(va) | TD_PA(pa) |
-		    TD_L | TD_CP | TD_CV | TD_P | TD_W;
-		tlb_store_slot(TLB_DTLB, va, TLB_CTX_KERNEL, tte,
-		    TLB_SLOT_TSB_KERNEL_MIN + i);
-	}
+	pmap_map_tsb();
 	bzero(tsb_kernel, KVA_PAGES * PAGE_SIZE_4M);
 
 	/*
-	 * Load the tsb registers.
+	 * Allocate a kernel stack with guard page for thread0 and map it into
+	 * the kernel tsb.
 	 */
-	stxa(AA_DMMU_TSB, ASI_DMMU, (vm_offset_t)tsb_kernel);
-	stxa(AA_IMMU_TSB, ASI_IMMU, (vm_offset_t)tsb_kernel);
-	membar(Sync);
-	flush(tsb_kernel);
+	pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE);
+	kstack0_phys = pa;
+	kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE);
+	virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE;
+	for (i = 0; i < KSTACK_PAGES; i++) {
+		pa = kstack0_phys + i * PAGE_SIZE;
+		va = kstack0 + i * PAGE_SIZE;
+		pmap_kenter(va, pa);
+		tlb_page_demap(TLB_DTLB, TLB_CTX_KERNEL, va);
+	}
 
 	/*
 	 * Allocate the message buffer.
@@ -399,6 +399,36 @@ pmap_bootstrap(vm_offset_t ekva)
 	pm->pm_active = ~0;
 	pm->pm_count = 1;
 	TAILQ_INIT(&pm->pm_pvlist);
+}
+
+void
+pmap_map_tsb(void)
+{
+	struct tte tte;
+	vm_offset_t va;
+	vm_offset_t pa;
+	int i;
+
+	/*
+	 * Map the 4mb tsb pages.
+	 */
+	for (i = 0; i < KVA_PAGES; i++) {
+		va = (vm_offset_t)tsb_kernel + i * PAGE_SIZE_4M;
+		pa = tsb_kernel_phys + i * PAGE_SIZE_4M;
+		tte.tte_tag = TT_CTX(TLB_CTX_KERNEL) | TT_VA(va);
+		tte.tte_data = TD_V | TD_4M | TD_VA_LOW(va) | TD_PA(pa) |
+		    TD_L | TD_CP | TD_CV | TD_P | TD_W;
+		tlb_store_slot(TLB_DTLB, va, TLB_CTX_KERNEL, tte,
+		    TLB_SLOT_TSB_KERNEL_MIN + i);
+	}
+
+	/*
+	 * Load the tsb registers.
+	 */
+	stxa(AA_DMMU_TSB, ASI_DMMU, (vm_offset_t)tsb_kernel);
+	stxa(AA_IMMU_TSB, ASI_IMMU, (vm_offset_t)tsb_kernel);
+	membar(Sync);
+	flush(tsb_kernel);
 
 	/*
 	 * Set the secondary context to be the kernel context (needed for