- Search the whole OFW device tree instead of only the children of the

  root nexus device for the CPUs as starting with UltraSPARC IV the 'cpu'
  nodes hang off of from 'cmp' (chip multi-threading processor) or 'core'
  or combinations thereof. Also in large UltraSPARC III based machines
  the 'cpu' nodes hang off of 'ssm' (scalable shared memory) nodes which
  group snooping-coherency domains together instead of directly from the
  nexus.
  It would be great if we could use newbus to deal with the different ways
  the 'cpu' devices can hang off of pseudo ones but unfortunately both
  cpu_mp_setmaxid() and sparc64_init() have to work prior to regular device
  probing.
- Add support for UltraSPARC IV and IV+ CPUs. Due to the fact that these
  are multi-core each CPU has two Fireplane config registers and thus the
  module/target ID has to be determined differently so the one specific
  to a certain core is used. Similarly, starting with UltraSPARC IV the
  individual cores use a different property in the OFW device tree to
  indicate the CPU/core ID as it no longer is in coincidence with the
  shared slot/socket ID.
  This involves changing the MD KTR code to not directly read the UPA
  module ID either. We use the MID stored in the per-CPU data instead of
  calling cpu_get_mid() as a replacement in order prevent clobbering any
  registers as side-effect in the assembler version. This requires CATR()
  invocations from mp_startup() prior to mapping the per-CPU pages to be
  removed though.
  While at it additionally distinguish between CPUs with Fireplane and
  JBus interconnects as these also use slightly different sizes for the
  JBus/agent/module/target IDs.
- Make sparc64_shutdown_final() static as it's not used outside of
  machdep.c.

6 files changed, 201 insertions, 128 deletions

diff --git a/sys/sparc64/include/ktr.h b/sys/sparc64/include/ktr.h
index 0b7e96a..5948ba2 100644
--- a/sys/sparc64/include/ktr.h
+++ b/sys/sparc64/include/ktr.h
@@ -34,11 +34,9 @@
 
 #include <sys/ktr.h>
 
-#include <machine/upa.h>
-
 #ifndef LOCORE
 
-#define	KTR_CPU	UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG))
+#define	KTR_CPU	PCPU_GET(mid)
 
 #else
 
@@ -74,7 +72,7 @@ l2:	add	r2, 1, r3 ; \
 	add	r1, r2, r1 ; \
 	rd	%tick, r2 ; \
 	stx	r2, [r1 + KTR_TIMESTAMP] ; \
-	UPA_GET_MID(r2) ; \
+	lduw	[PCPU(MID)], r2 ; \
 	stw	r2, [r1 + KTR_CPU] ; \
 	stw	%g0, [r1 + KTR_LINE] ; \
 	stx	%g0, [r1 + KTR_FILE] ; \
@@ -84,7 +82,7 @@ l2:	add	r2, 1, r3 ; \
 #define CATR(mask, desc, r1, r2, r3, l1, l2, l3) \
 	set	mask, r1 ; \
 	TEST(ktr_mask, r1, r2, r2, l3) ; \
-	UPA_GET_MID(r1) ; \
+	lduw	[PCPU(MID)], r1 ; \
 	mov	1, r2 ; \
 	sllx	r2, r1, r1 ; \
 	TEST(ktr_cpumask, r1, r2, r3, l3) ; \
diff --git a/sys/sparc64/include/md_var.h b/sys/sparc64/include/md_var.h
index 69c6d69..592d980 100644
--- a/sys/sparc64/include/md_var.h
+++ b/sys/sparc64/include/md_var.h
@@ -47,6 +47,8 @@ extern	vm_paddr_t kstack0_phys;
 struct	pcpu;
 struct	md_utrap;
 
+const char *cpu_cpuid_prop(void);
+uint32_t cpu_get_mid(void);
 void	cpu_identify(u_long vers, u_int clock, u_int id);
 void	cpu_setregs(struct pcpu *pc);
 int	is_physical_memory(vm_paddr_t addr);
diff --git a/sys/sparc64/include/upa.h b/sys/sparc64/include/upa.h
index 3e56917..43531cf 100644
--- a/sys/sparc64/include/upa.h
+++ b/sys/sparc64/include/upa.h
@@ -26,25 +26,16 @@
  */
 
 #ifndef _MACHINE_UPA_H_
-#define _MACHINE_UPA_H_
+#define	_MACHINE_UPA_H_
 
 #define	UPA_MEMSTART	0x1c000000000UL
 #define	UPA_MEMEND	0x1ffffffffffUL
 
 #define	UPA_CR_MID_SHIFT	(17)
 #define	UPA_CR_MID_SIZE		(5)
-#define	UPA_CR_MID_MASK \
+#define	UPA_CR_MID_MASK							\
 	(((1 << UPA_CR_MID_SIZE) - 1) << UPA_CR_MID_SHIFT)
 
 #define	UPA_CR_GET_MID(cr)	((cr & UPA_CR_MID_MASK) >> UPA_CR_MID_SHIFT)
 
-#ifdef LOCORE
-
-#define	UPA_GET_MID(r1) \
-	ldxa	[%g0] ASI_UPA_CONFIG_REG, r1 ; \
-	srlx	r1, UPA_CR_MID_SHIFT, r1 ; \
-	and	r1, (1 << UPA_CR_MID_SIZE) - 1, r1
-
-#endif
-
 #endif /* _MACHINE_UPA_H_ */
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index 06f6ebc..eed7db8 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -89,10 +89,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/bus.h>
 #include <machine/cache.h>
 #include <machine/clock.h>
+#include <machine/cmt.h>
 #include <machine/cpu.h>
+#include <machine/fireplane.h>
 #include <machine/fp.h>
 #include <machine/fsr.h>
 #include <machine/intr_machdep.h>
+#include <machine/jbus.h>
 #include <machine/md_var.h>
 #include <machine/metadata.h>
 #include <machine/ofw_machdep.h>
@@ -143,11 +146,12 @@ static int cpu_use_vis = 1;
 cpu_block_copy_t *cpu_block_copy;
 cpu_block_zero_t *cpu_block_zero;
 
+static phandle_t find_bsp(phandle_t node, uint32_t bspid);
 void sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3,
     ofw_vec_t *vec);
-void sparc64_shutdown_final(void *dummy, int howto);
+static void sparc64_shutdown_final(void *dummy, int howto);
 
-static void cpu_startup(void *);
+static void cpu_startup(void *arg);
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 
 CTASSERT((1 << INT_SHIFT) == sizeof(int));
@@ -236,18 +240,94 @@ spinlock_exit(void)
 		wrpr(pil, td->td_md.md_saved_pil, 0);
 }
 
+static phandle_t
+find_bsp(phandle_t node, uint32_t bspid)
+{
+	char type[sizeof("cpu")];
+	phandle_t child;
+	uint32_t cpuid;
+
+	for (; node != 0; node = OF_peer(node)) {
+		child = OF_child(node);
+		if (child > 0) {
+			child = find_bsp(child, bspid);
+			if (child > 0)
+				return (child);
+		} else {
+			if (OF_getprop(node, "device_type", type,
+			    sizeof(type)) <= 0)
+				continue;
+			if (strcmp(type, "cpu") != 0)
+				continue;
+			if (OF_getprop(node, cpu_cpuid_prop(), &cpuid,
+			    sizeof(cpuid)) <= 0)
+				continue;
+			if (cpuid == bspid)
+				return (node);
+		}
+	}
+	return (0);
+}
+
+const char *
+cpu_cpuid_prop(void)
+{
+
+	switch (cpu_impl) {
+	case CPU_IMPL_SPARC64:
+	case CPU_IMPL_ULTRASPARCI:
+	case CPU_IMPL_ULTRASPARCII:
+	case CPU_IMPL_ULTRASPARCIIi:
+	case CPU_IMPL_ULTRASPARCIIe:
+		return ("upa-portid");
+	case CPU_IMPL_ULTRASPARCIII:
+	case CPU_IMPL_ULTRASPARCIIIp:
+	case CPU_IMPL_ULTRASPARCIIIi:
+	case CPU_IMPL_ULTRASPARCIIIip:
+		return ("portid");
+	case CPU_IMPL_ULTRASPARCIV:
+	case CPU_IMPL_ULTRASPARCIVp:
+		return ("cpuid");
+	default:
+		return ("");
+	}
+}
+
+uint32_t
+cpu_get_mid(void)
+{
+
+	switch (cpu_impl) {
+	case CPU_IMPL_SPARC64:
+	case CPU_IMPL_ULTRASPARCI:
+	case CPU_IMPL_ULTRASPARCII:
+	case CPU_IMPL_ULTRASPARCIIi:
+	case CPU_IMPL_ULTRASPARCIIe:
+		return (UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG)));
+	case CPU_IMPL_ULTRASPARCIII:
+	case CPU_IMPL_ULTRASPARCIIIp:
+		return (FIREPLANE_CR_GET_AID(ldxa(AA_FIREPLANE_CONFIG,
+		    ASI_FIREPLANE_CONFIG_REG)));
+	case CPU_IMPL_ULTRASPARCIIIi:
+	case CPU_IMPL_ULTRASPARCIIIip:
+		return (JBUS_CR_GET_JID(ldxa(0, ASI_JBUS_CONFIG_REG)));
+	case CPU_IMPL_ULTRASPARCIV:
+	case CPU_IMPL_ULTRASPARCIVp:
+		return (INTR_ID_GET_ID(ldxa(AA_INTR_ID, ASI_INTR_ID)));
+	default:
+		return (0);
+	}
+}
+
 void
 sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 {
-	char type[8];
 	char *env;
 	struct pcpu *pc;
 	vm_offset_t end;
 	vm_offset_t va;
 	caddr_t kmdp;
-	phandle_t child;
 	phandle_t root;
-	uint32_t portid;
 
 	end = 0;
 	kmdp = NULL;
@@ -319,7 +399,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	pc = (struct pcpu *)(pcpu0 + (PCPU_PAGES * PAGE_SIZE)) - 1;
 	pcpu_init(pc, 0, sizeof(struct pcpu));
 	pc->pc_addr = (vm_offset_t)pcpu0;
-	pc->pc_mid = UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG));
+	pc->pc_mid = cpu_get_mid();
 	pc->pc_tlb_ctx = TLB_CTX_USER_MIN;
 	pc->pc_tlb_ctx_min = TLB_CTX_USER_MIN;
 	pc->pc_tlb_ctx_max = TLB_CTX_USER_MAX;
@@ -328,24 +408,11 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	 * Determine the OFW node and frequency of the BSP (and ensure the
 	 * BSP is in the device tree in the first place).
 	 */
-	pc->pc_node = 0;
 	root = OF_peer(0);
-	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
-		if (OF_getprop(child, "device_type", type, sizeof(type)) <= 0)
-			continue;
-		if (strcmp(type, "cpu") != 0)
-			continue;
-		if (OF_getprop(child, cpu_impl < CPU_IMPL_ULTRASPARCIII ?
-		    "upa-portid" : "portid", &portid, sizeof(portid)) <= 0)
-			continue;
-		if (portid == pc->pc_mid) {
-			pc->pc_node = child;
-			break;
-		}
-	}
+	pc->pc_node = find_bsp(root, pc->pc_mid);
 	if (pc->pc_node == 0)
 		OF_exit();
-	if (OF_getprop(child, "clock-frequency", &pc->pc_clock,
+	if (OF_getprop(pc->pc_node, "clock-frequency", &pc->pc_clock,
 	    sizeof(pc->pc_clock)) <= 0)
 		OF_exit();
 
@@ -838,7 +905,7 @@ cpu_halt(void)
 	cpu_shutdown(&args);
 }
 
-void
+static void
 sparc64_shutdown_final(void *dummy, int howto)
 {
 	static struct {
diff --git a/sys/sparc64/sparc64/mp_locore.S b/sys/sparc64/sparc64/mp_locore.S
index b089a2b..96239d5 100644
--- a/sys/sparc64/sparc64/mp_locore.S
+++ b/sys/sparc64/sparc64/mp_locore.S
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/ktr.h>
 #include <machine/pstate.h>
 #include <machine/smp.h>
-#include <machine/upa.h>
 #include <machine/ver.h>
 
 #include "assym.s"
@@ -212,14 +211,8 @@ ENTRY(mp_startup)
 	 nop
 	wr	%l1, 0, %asr24
 
-3:	UPA_GET_MID(%o0)
-
-#if KTR_COMPILE & KTR_SMP
-	CATR(KTR_SMP, "mp_start: CPU %d entered kernel"
-	    , %g1, %g2, %g3, 7, 8, 9)
-	stx	%o0, [%g1 + KTR_PARM1]
-9:
-#endif
+3:	call	cpu_get_mid
+	 nop
 
 	/*
 	 * Inform the boot processor we have inited.
@@ -236,13 +229,6 @@ ENTRY(mp_startup)
 	bne	%xcc, 4b
 	 nop
 
-#if KTR_COMPILE & KTR_SMP
-	CATR(KTR_SMP, "_mp_start: CPU %d got start signal"
-	    , %g1, %g2, %g3, 7, 8, 9)
-	stx	%o0, [%g1 + KTR_PARM1]
-9:
-#endif
-
 	add	%l0, CSA_TTES, %l1
 	clr	%l2
 
@@ -283,7 +269,7 @@ ENTRY(mp_startup)
 
 #if KTR_COMPILE & KTR_SMP
 	CATR(KTR_SMP,
-	    "_mp_start: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx"
+	    "mp_startup: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx"
 	    , %g1, %g2, %g3, 7, 8, 9)
 	lduw	[%l1 + PC_CPUID], %g2
 	stx	%g2, [%g1 + KTR_PARM1]
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index 997f0a4..8e9f9e1 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -119,7 +119,11 @@ static u_int cpuid_to_mid[MAXCPU];
 static int isjbus;
 static volatile u_int shutdown_cpus;
 
+static void ap_count(phandle_t node, u_int mid);
+static void ap_start(phandle_t node, u_int mid);
 static void cpu_mp_unleash(void *v);
+static void foreach_ap(phandle_t node, void (*func)(phandle_t node,
+    u_int mid));
 static void spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2);
 static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg);
 
@@ -166,25 +170,56 @@ mp_init(void)
 		cpu_ipi_selected = spitfire_ipi_selected;
 }
 
+static void
+foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid))
+{
+	char type[sizeof("cpu")];
+	phandle_t child;
+	u_int cpuid;
+ 
+	/* There's no need to traverse the whole OFW tree twice. */
+	if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1)
+		return;
+
+	for (; node != 0; node = OF_peer(node)) {
+		child = OF_child(node);
+		if (child > 0)
+			foreach_ap(child, func);
+		else {
+			if (OF_getprop(node, "device_type", type,
+			    sizeof(type)) <= 0)
+				continue;
+			if (strcmp(type, "cpu") != 0)
+				continue;
+			if (OF_getprop(node, cpu_cpuid_prop(), &cpuid,
+			    sizeof(cpuid)) <= 0)
+			panic("%s: can't get module ID", __func__);
+			if (cpuid == PCPU_GET(mid))
+				continue;
+			(*func)(node, cpuid);
+		}
+	}
+}
+
 /*
  * Probe for other CPUs.
  */
 void
-cpu_mp_setmaxid(void)
+cpu_mp_setmaxid()
 {
-	char buf[128];
-	phandle_t child;
-	u_int cpus;
 
 	all_cpus = 1 << curcpu;
 	mp_ncpus = 1;
+	mp_maxid = 0;
+
+	foreach_ap(OF_child(OF_peer(0)), ap_count);
+}
+
+static void
+ap_count(phandle_t node __unused, u_int mid __unused)
+{
 
-	cpus = 0;
-	for (child = OF_child(OF_peer(0)); child != 0; child = OF_peer(child))
-		if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 &&
-		    strcmp(buf, "cpu") == 0)
-			cpus++;
-	mp_maxid = cpus - 1;
+	mp_maxid++;
 }
 
 int
@@ -228,15 +263,6 @@ sun4u_startcpu(phandle_t cpu, void *func, u_long arg)
 void
 cpu_mp_start(void)
 {
-	char buf[128];
-	volatile struct cpu_start_args *csa;
-	struct pcpu *pc;
-	register_t s;
-	vm_offset_t va;
-	phandle_t child;
-	u_int mid;
-	u_int clock;
-	u_int cpuid;
 
 	mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN);
 
@@ -248,60 +274,7 @@ cpu_mp_start(void)
 
 	cpuid_to_mid[curcpu] = PCPU_GET(mid);
 
-	csa = &cpu_start_args;
-	for (child = OF_child(OF_peer(0)); child != 0 && mp_ncpus <= MAXCPU;
-	    child = OF_peer(child)) {
-		if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 ||
-		    strcmp(buf, "cpu") != 0)
-			continue;
-		if (OF_getprop(child, cpu_impl < CPU_IMPL_ULTRASPARCIII ?
-		    "upa-portid" : "portid", &mid, sizeof(mid)) <= 0)
-			panic("%s: can't get module ID", __func__);
-		if (mid == PCPU_GET(mid))
-			continue;
-		if (OF_getprop(child, "clock-frequency", &clock,
-		    sizeof(clock)) <= 0)
-			panic("%s: can't get clock", __func__);
-		if (clock != PCPU_GET(clock))
-			hardclock_use_stick = 1;
-
-		csa->csa_state = 0;
-		sun4u_startcpu(child, (void *)mp_tramp, 0);
-		s = intr_disable();
-		while (csa->csa_state != CPU_TICKSYNC)
-			;
-		membar(StoreLoad);
-		csa->csa_tick = rd(tick);
-		if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
-			while (csa->csa_state != CPU_STICKSYNC)
-				;
-			membar(StoreLoad);
-			csa->csa_stick = rdstick();
-		}
-		while (csa->csa_state != CPU_INIT)
-			;
-		csa->csa_tick = csa->csa_stick = 0;
-		intr_restore(s);
-
-		cpuid = mp_ncpus++;
-		cpuid_to_mid[cpuid] = mid;
-		cpu_identify(csa->csa_ver, clock, cpuid);
-
-		va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
-		pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
-		pcpu_init(pc, cpuid, sizeof(*pc));
-		dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE),
-		    cpuid);
-		pc->pc_addr = va;
-		pc->pc_clock = clock;
-		pc->pc_mid = mid;
-		pc->pc_node = child;
-
-		cache_init(pc);
-
-		all_cpus |= 1 << cpuid;
-		intr_add_cpu(cpuid);
-	}
+	foreach_ap(OF_child(OF_peer(0)), ap_start);
 	KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS,
 	    ("%s: can only IPI a maximum of %d JBus-CPUs",
 	    __func__, IDR_JALAPENO_MAX_BN_PAIRS));
@@ -309,6 +282,62 @@ cpu_mp_start(void)
 	smp_active = 1;
 }
 
+static void
+ap_start(phandle_t node, u_int mid)
+{
+	volatile struct cpu_start_args *csa;
+	struct pcpu *pc;
+	register_t s;
+	vm_offset_t va;
+	u_int clock;
+	u_int cpuid;
+
+	if (mp_ncpus > MAXCPU)
+		return;
+
+	if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0)
+		panic("%s: can't get clock", __func__);
+	if (clock != PCPU_GET(clock))
+		hardclock_use_stick = 1;
+
+	csa = &cpu_start_args;
+	csa->csa_state = 0;
+	sun4u_startcpu(node, (void *)mp_tramp, 0);
+	s = intr_disable();
+	while (csa->csa_state != CPU_TICKSYNC)
+		;
+	membar(StoreLoad);
+	csa->csa_tick = rd(tick);
+	if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
+		while (csa->csa_state != CPU_STICKSYNC)
+			;
+		membar(StoreLoad);
+		csa->csa_stick = rdstick();
+	}
+	while (csa->csa_state != CPU_INIT)
+		;
+	csa->csa_tick = csa->csa_stick = 0;
+	intr_restore(s);
+
+	cpuid = mp_ncpus++;
+	cpuid_to_mid[cpuid] = mid;
+	cpu_identify(csa->csa_ver, clock, cpuid);
+
+	va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE);
+	pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1;
+	pcpu_init(pc, cpuid, sizeof(*pc));
+	dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE), cpuid);
+	pc->pc_addr = va;
+	pc->pc_clock = clock;
+	pc->pc_mid = mid;
+	pc->pc_node = node;
+
+	cache_init(pc);
+
+	all_cpus |= 1 << cpuid;
+	intr_add_cpu(cpuid);
+}
+
 void
 cpu_mp_announce(void)
 {