This pushes all of JC's patches that I have in place. I

am now able to run 32 cores ok.. but I still will hang
on buildworld with a NFS problem. I suspect I am missing
a patch for the netlogic rge driver.

JC check and see if I am missing anything except your
core-mask changes

Obtained from:	JC

19 files changed, 496 insertions, 612 deletions

diff --git a/sys/mips/conf/XLR b/sys/mips/conf/XLR
index a0737b3..9daa356 100644
--- a/sys/mips/conf/XLR
+++ b/sys/mips/conf/XLR
@@ -59,10 +59,11 @@ include		"../rmi/std.xlr"
 makeoptions	DEBUG=-g		# Build kernel with gdb(1) debug symbols
 #profile		2
 
-#options 	SCHED_ULE		# ULE scheduler
+options 	SCHED_ULE		# ULE scheduler
 #options		VERBOSE_SYSINIT
-options 	SCHED_4BSD		# 4BSD scheduler
-#options 	PREEMPTION		# Enable kernel thread preemption
+#options 	SCHED_4BSD		# 4BSD scheduler
+options         SMP
+options 	PREEMPTION		# Enable kernel thread preemption
 #options 	FULL_PREEMPTION		# Enable kernel thread preemption
 options 	INET			# InterNETworking
 options 	INET6			# IPv6 communications protocols
diff --git a/sys/mips/include/locore.h b/sys/mips/include/locore.h
index b1b9f59..7cc72c7 100644
--- a/sys/mips/include/locore.h
+++ b/sys/mips/include/locore.h
@@ -60,6 +60,7 @@ typedef int mips_prid_t;
 				/*	0x09	unannounced */
 				/*	0x0a	unannounced */
 #define     MIPS_PRID_CID_LEXRA		0x0b	/* Lexra */
+#define     MIPS_PRID_CID_RMI		0x0c	/* RMI */
 #define     MIPS_PRID_CID_CAVIUM	0x0d	/* Cavium */
 #define MIPS_PRID_COPTS(x)	(((x) >> 24) & 0x00ff)	/* Company Options */
 
diff --git a/sys/mips/include/param.h b/sys/mips/include/param.h
index 06cdeac..7eae10a 100644
--- a/sys/mips/include/param.h
+++ b/sys/mips/include/param.h
@@ -70,7 +70,7 @@
 #define	MID_MACHINE	0	/* None but has to be defined */
 
 #ifdef SMP
-#define	MAXSMPCPU	16
+#define	MAXSMPCPU	32
 #define	MAXCPU		MAXSMPCPU
 #else
 #define	MAXSMPCPU	1
diff --git a/sys/mips/include/smp.h b/sys/mips/include/smp.h
index 1cb0596..216b9e7 100644
--- a/sys/mips/include/smp.h
+++ b/sys/mips/include/smp.h
@@ -26,6 +26,7 @@
 #define	IPI_AST			0x0004
 #define	IPI_STOP		0x0008
 #define	IPI_STOP_HARD		0x0008
+#define	IPI_PREEMPT		0x0010
 
 #ifndef LOCORE
 
diff --git a/sys/mips/mips/cpu.c b/sys/mips/mips/cpu.c
index 4539d1e..7c61fb8 100644
--- a/sys/mips/mips/cpu.c
+++ b/sys/mips/mips/cpu.c
@@ -178,6 +178,9 @@ cpu_identify(void)
 	case MIPS_PRID_CID_LEXRA:
 		printf("Lexra");
 		break;
+	case MIPS_PRID_CID_RMI:
+		printf("RMI");
+		break;
 	case MIPS_PRID_CID_CAVIUM:
 		printf("Cavium");
 		break;
diff --git a/sys/mips/mips/machdep.c b/sys/mips/mips/machdep.c
index a9c484c..59a7656 100644
--- a/sys/mips/mips/machdep.c
+++ b/sys/mips/mips/machdep.c
@@ -346,7 +346,7 @@ mips_vector_init(void)
 	bcopy(MipsTLBMiss, (void *)TLB_MISS_EXC_VEC,
 	      MipsTLBMissEnd - MipsTLBMiss);
 
-#ifdef TARGET_OCTEON
+#if defined(TARGET_OCTEON) || defined(TARGET_XLR_XLS)
 /* Fake, but sufficient, for the 32-bit with 64-bit hardware addresses  */
 	bcopy(MipsTLBMiss, (void *)XTLB_MISS_EXC_VEC,
 	      MipsTLBMissEnd - MipsTLBMiss);
diff --git a/sys/mips/mips/mp_machdep.c b/sys/mips/mips/mp_machdep.c
index f0c142f..3278d73 100644
--- a/sys/mips/mips/mp_machdep.c
+++ b/sys/mips/mips/mp_machdep.c
@@ -141,6 +141,10 @@ mips_ipi_handler(void *arg)
 			atomic_clear_int(&stopped_cpus, cpumask);
 			CTR0(KTR_SMP, "IPI_STOP (restart)");
 			break;
+		case IPI_PREEMPT:
+			CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
+			sched_preempt(curthread);
+			break;
 		default:
 			panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu);
 		}
@@ -234,7 +238,9 @@ cpu_mp_start(void)
 void
 smp_init_secondary(u_int32_t cpuid)
 {
+#ifndef TARGET_XLR_XLS
 	int ipi_int_mask, clock_int_mask;
+#endif
 
 	/* TLB */
 	Mips_SetWIRED(0);
@@ -288,12 +294,16 @@ smp_init_secondary(u_int32_t cpuid)
 	while (smp_started == 0)
 		; /* nothing */
 
+#ifndef TARGET_XLR_XLS
 	/*
 	 * Unmask the clock and ipi interrupts.
 	 */
 	clock_int_mask = hard_int_mask(5);
 	ipi_int_mask = hard_int_mask(platform_ipi_intrnum());
 	set_intr_mask(ALL_INT_MASK & ~(ipi_int_mask | clock_int_mask));
+#else
+	platform_init_ap(cpuid);
+#endif
 
 	/*
 	 * Bootstrap the compare register.
diff --git a/sys/mips/mips/mpboot.S b/sys/mips/mips/mpboot.S
index 631099c..89a1dd3 100644
--- a/sys/mips/mips/mpboot.S
+++ b/sys/mips/mips/mpboot.S
@@ -76,9 +76,10 @@ GLOBAL(mpentry)
 
 	PTR_LA	gp, _C_LABEL(_gp)
 
+#ifndef TARGET_XLR_XLS
 	jal	platform_init_ap
 	move	a0, s0
-
+#endif
 	jal	smp_init_secondary
 	move	a0, s0
 
diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c
index bcca62e..6d0c02b 100644
--- a/sys/mips/mips/pmap.c
+++ b/sys/mips/mips/pmap.c
@@ -80,6 +80,7 @@ __FBSDID("$FreeBSD$");
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
+#include <vm/vm_phys.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <vm/vm_kern.h>
@@ -109,6 +110,10 @@ __FBSDID("$FreeBSD$");
 #define	PMAP_SHPGPERPROC 200
 #endif
 
+#if defined(TARGET_XLR_XLS)
+#define HIGHMEM_SUPPORT
+#endif
+
 #if !defined(PMAP_DIAGNOSTIC)
 #define	PMAP_INLINE __inline
 #else
@@ -183,12 +188,18 @@ static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t);
 static int init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot);
 static void pmap_TLB_invalidate_kernel(vm_offset_t);
 static void pmap_TLB_update_kernel(vm_offset_t, pt_entry_t);
+static vm_page_t pmap_alloc_pte_page(pmap_t, unsigned int, int, vm_offset_t *);
+static void pmap_release_pte_page(vm_page_t);
 
 #ifdef SMP
 static void pmap_invalidate_page_action(void *arg);
 static void pmap_invalidate_all_action(void *arg);
 static void pmap_update_page_action(void *arg);
+#endif
 
+#ifdef HIGHMEM_SUPPORT
+static void * pmap_ptpgzone_allocf(uma_zone_t, int, u_int8_t*, int);
+static uma_zone_t ptpgzone;
 #endif
 
 struct local_sysmaps {
@@ -530,6 +541,12 @@ pmap_init(void)
 	pv_entry_max = PMAP_SHPGPERPROC * maxproc + cnt.v_page_count;
 	pv_entry_high_water = 9 * (pv_entry_max / 10);
 	uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max);
+
+#ifdef HIGHMEM_SUPPORT
+	ptpgzone = uma_zcreate("PT ENTRY", PAGE_SIZE, NULL,
+	    NULL, NULL, NULL, PAGE_SIZE-1, UMA_ZONE_NOFREE);
+	uma_zone_set_allocf(ptpgzone, pmap_ptpgzone_allocf);
+#endif
 }
 
 /***************************************************
@@ -887,7 +904,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
 	/*
 	 * If the page is finally unwired, simply free it.
 	 */
-	vm_page_free_zero(m);
+	pmap_release_pte_page(m);
 	atomic_subtract_int(&cnt.v_wire_count, 1);
 	return (1);
 }
@@ -947,6 +964,118 @@ pmap_pinit0(pmap_t pmap)
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 }
 
+#ifdef HIGHMEM_SUPPORT
+static void *
+pmap_ptpgzone_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
+{
+	vm_page_t m;
+	vm_paddr_t paddr;
+
+	*flags = UMA_SLAB_PRIV;
+	m = vm_phys_alloc_contig(1, 0, MIPS_KSEG0_LARGEST_PHYS,
+	     PAGE_SIZE, PAGE_SIZE);
+	if (m == NULL)
+		return (NULL);
+
+	paddr = VM_PAGE_TO_PHYS(m);
+	return ((void *)MIPS_PHYS_TO_KSEG0(paddr));
+}	
+
+static vm_page_t
+pmap_alloc_pte_page(pmap_t pmap, unsigned int index, int wait, vm_offset_t *vap)
+{
+	vm_paddr_t paddr;
+	void *va;
+	vm_page_t m;
+	int locked;
+
+	locked = mtx_owned(&pmap->pm_mtx);
+	if (locked) {
+		mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+		PMAP_UNLOCK(pmap);
+		vm_page_unlock_queues();
+	}
+	va = uma_zalloc(ptpgzone, wait);
+	if (locked) {
+		vm_page_lock_queues();
+		PMAP_LOCK(pmap);
+	}
+	if (va == NULL)
+		return (NULL);
+
+	paddr = MIPS_KSEG0_TO_PHYS(va);
+	m = PHYS_TO_VM_PAGE(paddr);
+	
+	if ((m->flags & PG_ZERO) == 0)
+		bzero(va, PAGE_SIZE);
+	m->pindex = index;
+	m->valid = VM_PAGE_BITS_ALL;
+	m->wire_count = 1;
+	atomic_add_int(&cnt.v_wire_count, 1);
+	*vap = (vm_offset_t)va;
+	return (m);
+}
+
+static void
+pmap_release_pte_page(vm_page_t m)
+{
+	void *va;
+	vm_paddr_t paddr;
+
+	paddr = VM_PAGE_TO_PHYS(m);
+	va = (void *)MIPS_PHYS_TO_KSEG0(paddr);
+	uma_zfree(ptpgzone, va);
+}
+#else
+static vm_page_t
+pmap_alloc_pte_page(pmap_t pmap, unsigned int index, int wait, vm_offset_t *vap)
+{
+	vm_offset_t va;
+	vm_page_t m;
+	int locked, req;
+
+	locked = mtx_owned(&pmap->pm_mtx);
+	req = VM_ALLOC_WIRED | VM_ALLOC_NOOBJ;
+	if (wait & M_WAITOK)
+		req |= VM_ALLOC_NORMAL;
+	else
+		req |= VM_ALLOC_INTERRUPT;
+
+	m = vm_page_alloc(NULL, index, req);
+	if (m == NULL) {
+		if (wait & M_WAITOK) {
+			if (locked) {
+				mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+				PMAP_UNLOCK(pmap);
+				vm_page_unlock_queues();
+			}
+			VM_WAIT;
+			if (locked) {
+				vm_page_lock_queues();
+				PMAP_LOCK(pmap);
+			}
+		}
+		return NULL;
+	}
+	
+	va = MIPS_PHYS_TO_KSEG0(VM_PAGE_TO_PHYS(m));
+	if ((m->flags & PG_ZERO) == 0)
+		bzero((void *)va, PAGE_SIZE);
+	else
+		vm_page_flag_clear(m, PG_ZERO);
+	
+	m->valid = VM_PAGE_BITS_ALL;
+	*vap = (vm_offset_t)va;
+	return (m);
+}
+
+static void
+pmap_release_pte_page(vm_page_t m)
+{
+	vm_page_free(m);
+}
+#endif
+
 /*
  * Initialize a preallocated and zeroed pmap structure,
  * such as one in a vmspace structure.
@@ -955,37 +1084,16 @@ int
 pmap_pinit(pmap_t pmap)
 {
 	vm_offset_t ptdva;
-	vm_paddr_t ptdpa;
 	vm_page_t ptdpg;
 	int i;
-	int req;
 
 	PMAP_LOCK_INIT(pmap);
 
-	req = VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
-	    VM_ALLOC_ZERO;
-
 	/*
 	 * allocate the page directory page
 	 */
-	while ((ptdpg = vm_page_alloc(NULL, NUSERPGTBLS, req)) == NULL)
-		VM_WAIT;
-
-	ptdpg->valid = VM_PAGE_BITS_ALL;
-
-	ptdpa = VM_PAGE_TO_PHYS(ptdpg);
-	if (ptdpa < MIPS_KSEG0_LARGEST_PHYS) {
-		ptdva = MIPS_PHYS_TO_KSEG0(ptdpa);
-	} else {
-		ptdva = kmem_alloc_nofault(kernel_map, PAGE_SIZE);
-		if (ptdva == 0)
-			panic("pmap_pinit: unable to allocate kva");
-		pmap_kenter(ptdva, ptdpa);
-	}
-
+	ptdpg = pmap_alloc_pte_page(pmap, NUSERPGTBLS, M_WAITOK, &ptdva);
 	pmap->pm_segtab = (pd_entry_t *)ptdva;
-	if ((ptdpg->flags & PG_ZERO) == 0)
-		bzero(pmap->pm_segtab, PAGE_SIZE);
 
 	pmap->pm_active = 0;
 	pmap->pm_ptphint = NULL;
@@ -1006,7 +1114,7 @@ pmap_pinit(pmap_t pmap)
 static vm_page_t
 _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags)
 {
-	vm_offset_t pteva, ptepa;
+	vm_offset_t pteva;
 	vm_page_t m;
 	int req;
 
@@ -1018,25 +1126,9 @@ _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags)
 	/*
 	 * Find or fabricate a new pagetable page
 	 */
-	if ((m = vm_page_alloc(NULL, ptepindex, req)) == NULL) {
-		if (flags & M_WAITOK) {
-			PMAP_UNLOCK(pmap);
-			vm_page_unlock_queues();
-			VM_WAIT;
-			vm_page_lock_queues();
-			PMAP_LOCK(pmap);
-		}
-		/*
-		 * Indicate the need to retry.	While waiting, the page
-		 * table page may have been allocated.
-		 */
+	m = pmap_alloc_pte_page(pmap, ptepindex, flags, &pteva);
+	if (m == NULL)
 		return (NULL);
-	}
-	if ((m->flags & PG_ZERO) == 0)
-		pmap_zero_page(m);
-
-	KASSERT(m->queue == PQ_NONE,
-	    ("_pmap_allocpte: %p->queue != PQ_NONE", m));
 
 	/*
 	 * Map the pagetable page into the process address space, if it
@@ -1044,34 +1136,12 @@ _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags)
 	 */
 
 	pmap->pm_stats.resident_count++;
-
-	ptepa = VM_PAGE_TO_PHYS(m);
-	if (ptepa < MIPS_KSEG0_LARGEST_PHYS) {
-		pteva = MIPS_PHYS_TO_KSEG0(ptepa);
-	} else {
-		pteva = kmem_alloc_nofault(kernel_map, PAGE_SIZE);
-		if (pteva == 0)
-			panic("_pmap_allocpte: unable to allocate kva");
-		pmap_kenter(pteva, ptepa);
-	}
-
 	pmap->pm_segtab[ptepindex] = (pd_entry_t)pteva;
 
 	/*
 	 * Set the page table hint
 	 */
 	pmap->pm_ptphint = m;
-
-	/*
-	 * Kernel page tables are allocated in pmap_bootstrap() or
-	 * pmap_growkernel().
-	 */
-	if (is_kernel_pmap(pmap))
-		panic("_pmap_allocpte() called for kernel pmap\n");
-
-	m->valid = VM_PAGE_BITS_ALL;
-	vm_page_flag_clear(m, PG_ZERO);
-
 	return (m);
 }
 
@@ -1158,17 +1228,12 @@ pmap_release(pmap_t pmap)
 	ptdva = (vm_offset_t)pmap->pm_segtab;
 	ptdpg = PHYS_TO_VM_PAGE(vtophys(ptdva));
 
-	if (ptdva >= VM_MIN_KERNEL_ADDRESS) {
-		pmap_kremove(ptdva);
-		kmem_free(kernel_map, ptdva, PAGE_SIZE);
-	} else {
-		KASSERT(MIPS_IS_KSEG0_ADDR(ptdva),
-		    ("pmap_release: 0x%0lx is not in kseg0", (long)ptdva));
-	}
+	KASSERT(MIPS_IS_KSEG0_ADDR(ptdva),
+	    ("pmap_release: 0x%0lx is not in kseg0", (long)ptdva));
 
 	ptdpg->wire_count--;
 	atomic_subtract_int(&cnt.v_wire_count, 1);
-	vm_page_free_zero(ptdpg);
+	pmap_release_pte_page(ptdpg);
 	PMAP_LOCK_DESTROY(pmap);
 }
 
@@ -1178,10 +1243,10 @@ pmap_release(pmap_t pmap)
 void
 pmap_growkernel(vm_offset_t addr)
 {
-	vm_offset_t ptppaddr;
+	vm_offset_t pageva;
 	vm_page_t nkpg;
 	pt_entry_t *pte;
-	int i, req;
+	int i;
 
 	mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 	if (kernel_vm_end == 0) {
@@ -1213,26 +1278,13 @@ pmap_growkernel(vm_offset_t addr)
 		/*
 		 * This index is bogus, but out of the way
 		 */
-		req = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ;
-		nkpg = vm_page_alloc(NULL, nkpt, req);
+		nkpg = pmap_alloc_pte_page(kernel_pmap, nkpt, M_NOWAIT, &pageva);
+
 		if (!nkpg)
 			panic("pmap_growkernel: no memory to grow kernel");
 
 		nkpt++;
-
-		ptppaddr = VM_PAGE_TO_PHYS(nkpg);
-		if (ptppaddr >= MIPS_KSEG0_LARGEST_PHYS) {
-			/*
-			 * We need to do something here, but I am not sure
-			 * what. We can access anything in the 0 - 512Meg
-			 * region, but if we get a page to go in the kernel
-			 * segmap that is outside of of that we really need
-			 * to have another mapping beyond the temporary ones
-			 * I have. Not sure how to do this yet. FIXME FIXME.
-			 */
-			panic("Gak, can't handle a k-page table outside of lower 512Meg");
-		}
-		pte = (pt_entry_t *)MIPS_PHYS_TO_KSEG0(ptppaddr);
+		pte = (pt_entry_t *)pageva;
 		segtab_pde(kernel_segmap, kernel_vm_end) = (pd_entry_t)pte;
 
 		/*
@@ -1396,7 +1448,8 @@ pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va)
 		}
 	}
 
-	KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
+	KASSERT(pv != NULL, ("pmap_remove_entry: pv not found, pa %lx va %lx",
+	     (u_long)VM_PAGE_TO_PHYS(m), (u_long)va));
 	TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 	m->md.pv_list_count--;
 	if (TAILQ_FIRST(&m->md.pv_list) == NULL)
diff --git a/sys/mips/rmi/dev/xlr/rge.c b/sys/mips/rmi/dev/xlr/rge.c
index c62c0de..0a015ac 100644
--- a/sys/mips/rmi/dev/xlr/rge.c
+++ b/sys/mips/rmi/dev/xlr/rge.c
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #define __RMAN_RESOURCE_VISIBLE
 #include <sys/rman.h>
 #include <sys/taskqueue.h>
+#include <sys/smp.h>
 
 #include <net/if.h>
 #include <net/if_arp.h>
@@ -83,6 +84,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/param.h>
 #include <machine/intr_machdep.h>
 #include <machine/clock.h>	/* for DELAY */
+#include <machine/cpuregs.h>
 #include <machine/bus.h>	/* */
 #include <machine/resource.h>
 #include <mips/rmi/interrupt.h>
@@ -112,7 +114,6 @@ MODULE_DEPEND(rge, ether, 1, 1, 1);
 MODULE_DEPEND(rge, miibus, 1, 1, 1);
 
 /* #define DEBUG */
-/*#define RX_COPY */
 
 #define RGE_TX_THRESHOLD 1024
 #define RGE_TX_Q_SIZE 1024
@@ -204,10 +205,18 @@ ldadd_wu(unsigned int value, unsigned long *addr)
 	return value;
 }
 
+static __inline__ uint32_t 
+xlr_enable_kx(void)
+{
+	uint32_t sr = mips_rd_status();
+
+	mips_wr_status((sr & ~MIPS_SR_INT_IE) | MIPS_SR_KX);
+	return sr;
+}
+
 /* #define mac_stats_add(x, val) ({(x) += (val);}) */
 #define mac_stats_add(x, val) ldadd_wu(val, &x)
 
-
 #define XLR_MAX_CORE 8
 #define RGE_LOCK_INIT(_sc, _name) \
   mtx_init(&(_sc)->rge_mtx, _name, MTX_NETWORK_LOCK, MTX_DEF)
@@ -332,56 +341,6 @@ DRIVER_MODULE(miibus, rge, miibus_driver, miibus_devclass, 0, 0);
 #define STR(x) __STR(x)
 #endif
 
-#define XKPHYS        0x8000000000000000
-/* -- No longer needed RRS
-static __inline__ uint32_t
-lw_40bit_phys(uint64_t phys, int cca)
-{
-	uint64_t addr;
-	uint32_t value = 0;
-	unsigned long flags;
-
-	addr = XKPHYS | ((uint64_t) cca << 59) | (phys & 0xfffffffffcULL);
-
-	enable_KX(flags);
-	__asm__ __volatile__(
-	            ".set push\n"
-	            ".set noreorder\n"
-	            ".set mips64\n"
-	            "lw    %0, 0(%1) \n"
-	            ".set pop\n"
-	    :       "=r"(value)
-	    :       "r"(addr));
-
-	disable_KX(flags);
-	return value;
-}
-*/
-/* -- No longer used RRS
-static __inline__ uint64_t
-ld_40bit_phys(uint64_t phys, int cca)
-{
-	uint64_t addr;
-	uint64_t value = 0;
-	unsigned long flags;
-
-
-	addr = XKPHYS | ((uint64_t) cca << 59) | (phys & 0xfffffffffcULL);
-	enable_KX(flags);
-	__asm__ __volatile__(
-	            ".set push\n"
-	            ".set noreorder\n"
-	            ".set mips64\n"
-	            "ld    %0, 0(%1) \n"
-	            ".set pop\n"
-	    :       "=r"(value)
-	    :       "r"(addr));
-
-	disable_KX(flags);
-	return value;
-}
-*/
-
 void *xlr_tx_ring_mem;
 
 struct tx_desc_node {
@@ -449,7 +408,7 @@ init_p2d_allocation(void)
 
 	for (i = 0; i < 32; i++) {
 		if (cpumask & (1 << i)) {
-			cpu = cpu_ltop_map[i];
+			cpu = i;
 			if (!active_core[cpu / 4]) {
 				active_core[cpu / 4] = 1;
 				xlr_total_active_core++;
@@ -507,7 +466,7 @@ get_p2d_desc(void)
 {
 	struct tx_desc_node *node;
 	struct p2d_tx_desc *tx_desc = NULL;
-	int cpu = xlr_cpu_id();
+	int cpu = xlr_core_id();
 
 	mtx_lock_spin(&tx_desc_lock[cpu]);
 	node = TAILQ_FIRST(&tx_frag_desc[cpu]);
@@ -527,7 +486,7 @@ static void
 free_p2d_desc(struct p2d_tx_desc *tx_desc)
 {
 	struct tx_desc_node *node;
-	int cpu = xlr_cpu_id();
+	int cpu = xlr_core_id();
 
 	mtx_lock_spin(&tx_desc_lock[cpu]);
 	node = TAILQ_FIRST(&free_tx_frag_desc[cpu]);
@@ -553,7 +512,7 @@ build_frag_list(struct mbuf *m_head, struct msgrng_msg *p2p_msg, struct p2d_tx_d
 	vm_offset_t taddr;
 	uint64_t fr_stid;
 
-	fr_stid = (xlr_cpu_id() << 3) + xlr_thr_id() + 4;
+	fr_stid = (xlr_core_id() << 3) + xlr_thr_id() + 4;
 
 	if (tx_desc == NULL)
 		return 1;
@@ -620,21 +579,6 @@ build_frag_list(struct mbuf *m_head, struct msgrng_msg *p2p_msg, struct p2d_tx_d
 static void
 release_tx_desc(struct msgrng_msg *msg, int rel_buf)
 {
-	/*
-	 * OLD code: vm_paddr_t paddr = msg->msg0 & 0xffffffffffULL;
-	 * uint64_t temp; struct p2d_tx_desc *tx_desc; struct mbuf *m;
-	 * 
-	 * paddr += (XLR_MAX_TX_FRAGS * sizeof(uint64_t)); *** In o32 we will
-	 * crash here ****** temp = ld_40bit_phys(paddr, 3); tx_desc =
-	 * (struct p2d_tx_desc *)((vm_offset_t)temp);
-	 * 
-	 * if (rel_buf) { paddr += sizeof(uint64_t);
-	 * 
-	 * temp = ld_40bit_phys(paddr, 3);
-	 * 
-	 * m = (struct mbuf *)((vm_offset_t)temp); m_freem(m); } printf("Call
-	 * fre_p2d_desc\n"); free_p2d_desc(tx_desc);
-	 */
 	struct p2d_tx_desc *tx_desc, *chk_addr;
 	struct mbuf *m;
 
@@ -652,52 +596,6 @@ release_tx_desc(struct msgrng_msg *msg, int rel_buf)
 	free_p2d_desc(tx_desc);
 }
 
-#ifdef RX_COPY
-#define RGE_MAX_NUM_DESC (6 * MAX_NUM_DESC)
-uint8_t *rge_rx_buffers[RGE_MAX_NUM_DESC];
-static struct mtx rge_rx_mtx;
-int g_rx_buf_head;
-
-static void
-init_rx_buf(void)
-{
-	int i;
-	uint8_t *buf, *start;
-	uint32_t size, *ptr;
-
-	mtx_init(&rge_rx_mtx, "xlr rx_desc", NULL, MTX_SPIN);
-
-	size = (RGE_MAX_NUM_DESC * (MAX_FRAME_SIZE + XLR_CACHELINE_SIZE));
-
-	start = (uint8_t *) contigmalloc(size, M_DEVBUF, M_NOWAIT | M_ZERO,
-	    0, 0xffffffff, XLR_CACHELINE_SIZE, 0);
-	if (start == NULL)
-		panic("NO RX BUFFERS");
-	buf = start;
-	size = (MAX_FRAME_SIZE + XLR_CACHELINE_SIZE);
-	for (i = 0; i < RGE_MAX_NUM_DESC; i++) {
-		buf = start + (i * size);
-		ptr = (uint32_t *) buf;
-		*ptr = (uint32_t) buf;
-		rge_rx_buffers[i] = buf + XLR_CACHELINE_SIZE;
-	}
-}
-
-static void *
-get_rx_buf(void)
-{
-	void *ptr = NULL;
-
-	mtx_lock_spin(&rge_rx_mtx);
-	if (g_rx_buf_head < RGE_MAX_NUM_DESC) {
-		ptr = (void *)rge_rx_buffers[g_rx_buf_head];
-		g_rx_buf_head++;
-	}
-	mtx_unlock_spin(&rge_rx_mtx);
-	return ptr;
-}
-
-#endif
 
 static struct mbuf *
 get_mbuf(void)
@@ -716,23 +614,16 @@ static void
 free_buf(vm_paddr_t paddr)
 {
 	struct mbuf *m;
-	uint32_t *temp;
-	uint32_t mag, um;
+	uint32_t mag, um, sr;
+
+	sr = xlr_enable_kx();
+	um = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE);
+	mag = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE + sizeof(uint32_t));
+	mips_wr_status(sr);
 
-	/*
-	 * This will crash I think. RRS temp = lw_40bit_phys((paddr -
-	 * XLR_CACHELINE_SIZE), 3); m = (struct mbuf *)temp;
-	 */
-	/*
-	 * This gets us a kseg0 address for the mbuf/magic on the ring but
-	 * we need to get the va to free the mbuf. This is stored at *temp;
-	 */
-	temp = (uint32_t *) MIPS_PHYS_TO_KSEG0(paddr - XLR_CACHELINE_SIZE);
-	um = temp[0];
-	mag = temp[1];
 	if (mag != 0xf00bad) {
-		printf("Something is wrong kseg:%p found mag:%x not 0xf00bad\n",
-		    temp, mag);
+		printf("Something is wrong kseg:%lx found mag:%x not 0xf00bad\n",
+		    (u_long)paddr, mag);
 		return;
 	}
 	m = (struct mbuf *)um;
@@ -743,19 +634,13 @@ free_buf(vm_paddr_t paddr)
 static void *
 get_buf(void)
 {
-#ifdef RX_COPY
-	return get_rx_buf();
-#else
 	struct mbuf *m_new = NULL;
-
+	unsigned int *md;
 #ifdef INVARIANTS
 	vm_paddr_t temp1, temp2;
-
 #endif
-	unsigned int *md;
 
 	m_new = get_mbuf();
-
 	if (m_new == NULL)
 		return NULL;
 
@@ -765,8 +650,6 @@ get_buf(void)
 	md[1] = 0xf00bad;
 	m_adj(m_new, XLR_CACHELINE_SIZE);
 
-
-	/* return (void *)m_new; */
 #ifdef INVARIANTS
 	temp1 = vtophys((vm_offset_t)m_new->m_data);
 	temp2 = vtophys((vm_offset_t)m_new->m_data + 1536);
@@ -774,7 +657,6 @@ get_buf(void)
 		panic("ALLOCED BUFFER IS NOT CONTIGUOUS\n");
 #endif
 	return (void *)m_new->m_data;
-#endif
 }
 
 /**********************************************************************
@@ -826,13 +708,13 @@ xlr_mac_send_fr(struct driver_data *priv,
 {
 	int stid = priv->rfrbucket;
 	struct msgrng_msg msg;
-	int vcpu = (xlr_cpu_id() << 2) + xlr_thr_id();
+	int vcpu = xlr_cpu_id();
 
 	mac_make_desc_rfr(&msg, addr);
 
 	/* Send the packet to MAC */
-	dbg_msg("mac_%d: Sending free packet %llx to stid %d\n",
-	    priv->instance, addr, stid);
+	dbg_msg("mac_%d: Sending free packet %lx to stid %d\n",
+	    priv->instance, (u_long)addr, stid);
 	if (priv->type == XLR_XGMAC) {
 		while (message_send(1, MSGRNG_CODE_XGMAC, stid, &msg));
 	} else {
@@ -1084,18 +966,25 @@ rmi_xlr_config_pde(struct driver_data *priv)
 	/* uint32_t desc_pack_ctrl = 0; */
 	uint32_t cpumask;
 
-	cpumask = PCPU_GET(cpumask) | PCPU_GET(other_cpus);
+	cpumask = PCPU_GET(cpumask);
+#ifdef SMP
+	/*
+         * rge may be called before SMP start in a BOOTP/NFSROOT
+         * setup. we will distribute packets to other cpus only when
+         * the SMP is started.
+	 */
+	if (smp_started)
+		cpumask |= PCPU_GET(other_cpus);
+#endif
 
-	for (i = 0; i < 32; i++) {
+	for (i = 0; i < MAXCPU; i++) {
 		if (cpumask & (1 << i)) {
-			cpu = cpu_ltop_map[i];
+			cpu = i;
 			bucket = ((cpu >> 2) << 3);
-			//|(cpu & 0x03);
 			bucket_map |= (1ULL << bucket);
-			dbg_msg("i=%d, cpu=%d, bucket = %d, bucket_map=%llx\n",
-			    i, cpu, bucket, bucket_map);
 		}
 	}
+	printf("rmi_xlr_config_pde: bucket_map=%llx\n", bucket_map);
 
 	/* bucket_map = 0x1; */
 	xlr_write_reg(priv->mmio, R_PDE_CLASS_0, (bucket_map & 0xffffffff));
@@ -1116,6 +1005,28 @@ rmi_xlr_config_pde(struct driver_data *priv)
 }
 
 static void
+rge_smp_update_pde(void *dummy __unused)
+{
+	int i;
+	struct driver_data *priv;
+	struct rge_softc *sc;
+
+	printf("Updating packet distribution for SMP\n");
+	for (i = 0; i < XLR_MAX_MACS; i++) {
+		sc = dev_mac[i];
+		if (!sc)
+			continue;
+		priv = &(sc->priv);
+		rmi_xlr_mac_set_enable(priv, 0);
+		rmi_xlr_config_pde(priv);
+		rmi_xlr_mac_set_enable(priv, 1);
+	}
+}
+
+SYSINIT(rge_smp_update_pde, SI_SUB_SMP, SI_ORDER_ANY, rge_smp_update_pde, NULL);
+
+
+static void
 rmi_xlr_config_parser(struct driver_data *priv)
 {
 	/*
@@ -1613,10 +1524,7 @@ retry:
 static void
 mac_frin_replenish(void *args /* ignored */ )
 {
-#ifdef RX_COPY
-	return;
-#else
-	int cpu = xlr_cpu_id();
+	int cpu = xlr_core_id();
 	int done = 0;
 	int i = 0;
 
@@ -1685,7 +1593,6 @@ mac_frin_replenish(void *args /* ignored */ )
 		if (done == XLR_MAX_MACS)
 			break;
 	}
-#endif
 }
 
 static volatile uint32_t g_tx_frm_tx_ok=0;
@@ -1716,8 +1623,8 @@ rmi_xlr_mac_msgring_handler(int bucket, int size, int code,
 	struct rge_softc *sc = NULL;
 	struct driver_data *priv = 0;
 	struct ifnet *ifp;
-	int cpu = xlr_cpu_id();
-	int vcpu = (cpu << 2) + xlr_thr_id();
+	int vcpu = xlr_cpu_id();
+	int cpu = xlr_core_id();
 
 	dbg_msg("mac: bucket=%d, size=%d, code=%d, stid=%d, msg0=%llx msg1=%llx\n",
 	    bucket, size, code, stid, msg->msg0, msg->msg1);
@@ -2098,80 +2005,18 @@ rge_release_resources(struct rge_softc *sc)
 uint32_t gmac_rx_fail[32];
 uint32_t gmac_rx_pass[32];
 
-#ifdef RX_COPY
-static void
-rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len)
-{
-	/*
-	 * struct mbuf *m = (struct mbuf *)*(unsigned int *)((char *)addr -
-	 * XLR_CACHELINE_SIZE);
-	 */
-	struct mbuf *m;
-	void *ptr;
-	uint32_t *temp;
-	struct ifnet *ifp = sc->rge_ifp;
-	unsigned long msgrng_flags;
-	int cpu = PCPU_GET(cpuid);
-
-
-	temp = (uint32_t *) MIPS_PHYS_TO_KSEG0(paddr - XLR_CACHELINE_SIZE);
-
-	ptr = (void *)(temp + XLR_CACHELINE_SIZE);
-	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
-	if (m != NULL) {
-		m->m_len = m->m_pkthdr.len = MCLBYTES;
-		m_copyback(m, 0, len + BYTE_OFFSET, ptr);
-		/* align the data */
-		m->m_data += BYTE_OFFSET;
-		m->m_pkthdr.len = m->m_len = len;
-		m->m_pkthdr.rcvif = ifp;
-		gmac_rx_pass[cpu]++;
-	} else {
-		gmac_rx_fail[cpu]++;
-	}
-	msgrng_access_enable(msgrng_flags);
-	xlr_mac_send_fr(&sc->priv, paddr, MAX_FRAME_SIZE);
-	msgrng_access_disable(msgrng_flags);
-
-#ifdef DUMP_PACKETS
-	{
-		int i = 0;
-		unsigned char *buf = (char *)m->m_data;
-
-		printf("Rx Packet: length=%d\n", len);
-		for (i = 0; i < 64; i++) {
-			if (i && (i % 16) == 0)
-				printf("\n");
-			printf("%02x ", buf[i]);
-		}
-		printf("\n");
-	}
-#endif
-
-
-	if (m) {
-		ifp->if_ipackets++;
-		(*ifp->if_input) (ifp, m);
-	}
-}
-
-#else
 static void
 rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len)
 {
-	/*
-	 * struct mbuf *m = (struct mbuf *)*(unsigned int *)((char *)addr -
-	 * XLR_CACHELINE_SIZE);
-	 */
 	struct mbuf *m;
-	uint32_t *temp, tm, mag;
-
+	uint32_t tm, mag, sr;
 	struct ifnet *ifp = sc->rge_ifp;
 
+	sr = xlr_enable_kx();
+	tm = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE);
+	mag = xlr_paddr_lw(paddr - XLR_CACHELINE_SIZE + sizeof(uint32_t));
+	mips_wr_status(sr);
 
-	temp = (uint32_t *) MIPS_PHYS_TO_KSEG0(paddr - XLR_CACHELINE_SIZE);
-	tm = temp[0];
-	mag = temp[1];
 	m = (struct mbuf *)tm;
 	if (mag != 0xf00bad) {
 		/* somebody else packet Error - FIXME in intialization */
@@ -2201,8 +2046,6 @@ rge_rx(struct rge_softc *sc, vm_paddr_t paddr, int len)
 	(*ifp->if_input) (ifp, m);
 }
 
-#endif
-
 static void
 rge_intr(void *arg)
 {
@@ -2268,8 +2111,8 @@ rge_start_locked(struct ifnet *ifp, int threshold)
 	int prepend_pkt = 0;
 	int i = 0;
 	struct p2d_tx_desc *tx_desc = NULL;
-	int cpu = xlr_cpu_id();
-	uint32_t vcpu = (cpu << 2) + xlr_thr_id();
+	int cpu = xlr_core_id();
+	uint32_t vcpu = xlr_cpu_id();
 
 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 		return;
@@ -2716,9 +2559,6 @@ mac_common_init(void)
 {
 	init_p2d_allocation();
 	init_tx_ring();
-#ifdef RX_COPY
-	init_rx_buf();
-#endif
 
 	if (xlr_board_info.is_xls) {
 		if (register_msgring_handler(TX_STN_GMAC0,
diff --git a/sys/mips/rmi/files.xlr b/sys/mips/rmi/files.xlr
index fdf5b27..4ed2439 100644
--- a/sys/mips/rmi/files.xlr
+++ b/sys/mips/rmi/files.xlr
@@ -9,6 +9,7 @@ mips/rmi/msgring_xls.c				standard
 mips/rmi/board.c				standard
 mips/rmi/on_chip.c				standard
 mips/rmi/intr_machdep.c			        standard
+mips/rmi/mpwait.S			        optional smp
 mips/rmi/xlr_i2c.c              		optional iic
 mips/rmi/uart_bus_xlr_iodi.c			optional uart 
 mips/rmi/uart_cpu_mips_xlr.c			optional uart 
diff --git a/sys/mips/rmi/interrupt.h b/sys/mips/rmi/interrupt.h
index 247dc9f..edb475a 100644
--- a/sys/mips/rmi/interrupt.h
+++ b/sys/mips/rmi/interrupt.h
@@ -32,12 +32,9 @@
 
 /* Defines for the IRQ numbers */
 
-#define IRQ_DUMMY_UART           2
-#define IRQ_IPI_SMP_FUNCTION     3
-#define IRQ_IPI_SMP_RESCHEDULE   4
-#define IRQ_REMOTE_DEBUG         5
-#define IRQ_MSGRING              6
-#define IRQ_TIMER                7
+#define IRQ_IPI			41  /* 8-39 are mapped by PIC intr 0-31 */
+#define IRQ_MSGRING             6
+#define IRQ_TIMER               7
 
 /*
  * XLR needs custom pre and post handlers for PCI/PCI-e interrupts
diff --git a/sys/mips/rmi/intr_machdep.c b/sys/mips/rmi/intr_machdep.c
index 8785ac7..93c11d2 100644
--- a/sys/mips/rmi/intr_machdep.c
+++ b/sys/mips/rmi/intr_machdep.c
@@ -157,29 +157,16 @@ cpu_intr(struct trapframe *tf)
 	for (i = sizeof(eirr) * 8 - 1; i >= 0; i--) {
 		if ((eirr & (1ULL << i)) == 0)
 			continue;
-#ifdef SMP
-		/* These are reserved interrupts */
-		if ((i == IPI_AST) || (i == IPI_RENDEZVOUS) || (i == IPI_STOP)
-		    || (i == IPI_SMP_CALL_FUNCTION)) {
-			write_c0_eirr64(1ULL << i);
-			pic_ack(i, 0);
-			smp_handle_ipi(tf, i);
-			pic_delayed_ack(i, 0);
-			continue;
-		}
-#ifdef XLR_PERFMON
-		if (i == IPI_PERFMON) {
-			write_c0_eirr64(1ULL << i);
-			pic_ack(i, 0);
-			xlr_perfmon_sampler(NULL);
-			pic_delayed_ack(i, 0);
-			continue;
-		}
-#endif
-#endif
-		ie = mips_intr_events[i];
-		/* atomic_add_long(mih->cntp, 1); */
 
+		ie = mips_intr_events[i];
+		/* Don't account special IRQs */
+		switch (i) {
+		case IRQ_IPI:
+		case IRQ_MSGRING:
+			break;
+		default:
+			mips_intrcnt_inc(mips_intr_counters[i]);
+		}
 		write_c0_eirr64(1ULL << i);
 		pic_ack(i, 0);
 		if (!ie || TAILQ_EMPTY(&ie->ie_handlers)) {
diff --git a/sys/mips/rmi/iodi.c b/sys/mips/rmi/iodi.c
index 134de9b..1e8a4d1 100644
--- a/sys/mips/rmi/iodi.c
+++ b/sys/mips/rmi/iodi.c
@@ -223,6 +223,7 @@ iodi_attach(device_t dev)
 	 */
 	device_add_child(dev, "uart", 0);
 	device_add_child(dev, "xlr_i2c", 0);
+	device_add_child(dev, "pcib", 0);
 
 	if (xlr_board_info.usb)
 		device_add_child(dev, "ehci", 0);
diff --git a/sys/mips/rmi/on_chip.c b/sys/mips/rmi/on_chip.c
index cfab1d1..c406592 100644
--- a/sys/mips/rmi/on_chip.c
+++ b/sys/mips/rmi/on_chip.c
@@ -38,8 +38,19 @@ __FBSDID("$FreeBSD$");
 #include <sys/limits.h>
 #include <sys/bus.h>
 
+#include <sys/ktr.h>
+#include <sys/kernel.h>
+#include <sys/kthread.h>
+#include <sys/proc.h>
+#include <sys/resourcevar.h>
+#include <sys/sched.h>
+#include <sys/unistd.h>
+#include <sys/sysctl.h>
+#include <sys/malloc.h>
+
 #include <machine/reg.h>
 #include <machine/cpu.h>
+#include <machine/hwfunc.h>
 #include <machine/mips_opcode.h>
 
 #include <machine/param.h>
@@ -62,6 +73,15 @@ struct tx_stn_handler {
 	void *dev_id;
 };
 
+struct msgring_ithread {
+	struct thread *i_thread;
+	u_int i_pending;
+	u_int i_flags;
+	int i_cpu;
+};
+
+struct msgring_ithread *msgring_ithreads[MAXCPU];
+
 /* globals */
 static struct tx_stn_handler tx_stn_handlers[MAX_TX_STNS];
 
@@ -91,8 +111,6 @@ static uint32_t msgring_thread_mask;
 
 uint32_t msgrng_msg_cycles = 0;
 
-int xlr_counters[MAXCPU][XLR_MAX_COUNTERS] __aligned(XLR_CACHELINE_SIZE);
-
 void xlr_msgring_handler(struct trapframe *);
 
 void 
@@ -103,10 +121,10 @@ xlr_msgring_cpu_init(void)
 	int id;
 	unsigned long flags;
 
-	/* if not thread 0 */
-	if (xlr_thr_id() != 0)
-		return;
-	id = xlr_cpu_id();
+	KASSERT(xlr_thr_id() == 0,
+		("xlr_msgring_cpu_init from non-zero thread\n"));
+
+	id = xlr_core_id();
 
 	bucket_sizes = xlr_board_info.bucket_sizes;
 	cc_config = xlr_board_info.credit_configs[id];
@@ -156,10 +174,6 @@ xlr_msgring_config(void)
 
 	msgring_watermark_count = 1;
 	msgring_thread_mask = 0x01;
-/*   printf("[%s]: int_type = 0x%x, pop_num_buckets=%d, pop_bucket_mask=%x" */
-/*          "watermark_count=%d, thread_mask=%x\n", __FUNCTION__, */
-/*          msgring_int_type, msgring_pop_num_buckets, msgring_pop_bucket_mask, */
-/*          msgring_watermark_count, msgring_thread_mask); */
 }
 
 void 
@@ -172,7 +186,6 @@ xlr_msgring_handler(struct trapframe *tf)
 	unsigned int bucket_empty_bm = 0;
 	unsigned int status = 0;
 
-	xlr_inc_counter(MSGRNG_INT);
 	/* TODO: not necessary to disable preemption */
 	msgrng_flags_save(mflags);
 
@@ -185,8 +198,6 @@ xlr_msgring_handler(struct trapframe *tf)
 			break;
 
 		for (bucket = 0; bucket < msgring_pop_num_buckets; bucket++) {
-			uint32_t cycles = 0;
-
 			if ((bucket_empty_bm & (1 << bucket)) /* empty */ )
 				continue;
 
@@ -194,10 +205,6 @@ xlr_msgring_handler(struct trapframe *tf)
 			if (status)
 				continue;
 
-			xlr_inc_counter(MSGRNG_MSG);
-			msgrng_msg_cycles = mips_rd_count();
-			cycles = msgrng_msg_cycles;
-
 			tx_stid = xlr_board_info.msgmap[rx_stid];
 
 			if (!tx_stn_handlers[tx_stid].action) {
@@ -211,17 +218,12 @@ xlr_msgring_handler(struct trapframe *tf)
 				    &msg, tx_stn_handlers[tx_stid].dev_id);
 				msgrng_flags_save(mflags);
 			}
-			xlr_set_counter(MSGRNG_MSG_CYCLES, (read_c0_count() - cycles));
 		}
 	}
 
 	xlr_set_counter(MSGRNG_EXIT_STATUS, msgrng_read_status());
 
 	msgrng_flags_restore(mflags);
-
-	//dbg_msg("OUT irq=%d\n", irq);
-
-	/* Call the msg callback */
 }
 
 void 
@@ -249,8 +251,110 @@ disable_msgring_int(void *arg)
 	msgrng_access_restore(&msgrng_lock, mflags);
 }
 
-extern void platform_prep_smp_launch(void);
-extern void msgring_process_fast_intr(void *arg);
+static int
+msgring_process_fast_intr(void *arg)
+{
+	int cpu = PCPU_GET(cpuid);
+	volatile struct msgring_ithread *it;
+	struct thread *td;
+
+	/* wakeup an appropriate intr_thread for processing this interrupt */
+	it = (volatile struct msgring_ithread *)msgring_ithreads[cpu];
+	KASSERT(it != NULL, ("No interrupt thread on cpu %d", cpu));
+	td = it->i_thread;
+
+	/*
+	 * Interrupt thread will enable the interrupts after processing all
+	 * messages
+	 */
+	disable_msgring_int(NULL);
+	atomic_store_rel_int(&it->i_pending, 1);
+	thread_lock(td);
+	if (TD_AWAITING_INTR(td)) {
+		TD_CLR_IWAIT(td);
+		sched_add(td, SRQ_INTR);
+	}
+	thread_unlock(td);
+	return FILTER_HANDLED;
+}
+
+static void
+msgring_process(void *arg)
+{
+	volatile struct msgring_ithread *ithd;
+	struct thread *td;
+	struct proc *p;
+
+	td = curthread;
+	p = td->td_proc;
+	ithd = (volatile struct msgring_ithread *)arg;
+	KASSERT(ithd->i_thread == td,
+	    ("%s:msg_ithread and proc linkage out of sync", __func__));
+
+	/* First bind this thread to the right CPU */
+	thread_lock(td);
+	sched_bind(td, ithd->i_cpu);
+	thread_unlock(td);
+
+	atomic_store_rel_ptr((volatile uintptr_t *)&msgring_ithreads[ithd->i_cpu],
+	     (uintptr_t)arg);
+	enable_msgring_int(NULL);
+	
+	while (1) {
+		while (ithd->i_pending) {
+			/*
+			 * This might need a full read and write barrier to
+			 * make sure that this write posts before any of the
+			 * memory or device accesses in the handlers.
+			 */
+			xlr_msgring_handler(NULL);
+			atomic_store_rel_int(&ithd->i_pending, 0);
+			enable_msgring_int(NULL);
+		}
+		if (!ithd->i_pending) {
+			thread_lock(td);
+			if (ithd->i_pending) {
+			  thread_unlock(td);
+			  continue;
+			}
+			sched_class(td, PRI_ITHD);
+			TD_SET_IWAIT(td);
+			mi_switch(SW_VOL, NULL);
+			thread_unlock(td);
+		}
+	}
+
+}
+
+static void 
+create_msgring_thread(int cpu)
+{
+	struct msgring_ithread *ithd;
+	struct thread *td;
+	struct proc *p;
+	int error;
+
+	/* Create kernel thread for message ring interrupt processing */
+	/* Currently create one task for thread 0 of each core */
+	ithd = malloc(sizeof(struct msgring_ithread),
+	    M_DEVBUF, M_WAITOK | M_ZERO);
+	error = kproc_create(msgring_process, (void *)ithd, &p,
+	    RFSTOPPED | RFHIGHPID, 2, "msg_intr%d", cpu);
+
+	if (error)
+		panic("kproc_create() failed with %d", error);
+	td = FIRST_THREAD_IN_PROC(p);	/* XXXKSE */
+
+	ithd->i_thread = td;
+	ithd->i_pending = 0;
+	ithd->i_cpu = cpu;
+
+	thread_lock(td);
+	sched_class(td, PRI_ITHD);
+	sched_add(td, SRQ_INTR);
+	thread_unlock(td);
+	CTR2(KTR_INTR, "%s: created %s", __func__, ithd_name[cpu]);
+}
 
 int 
 register_msgring_handler(int major,
@@ -272,14 +376,10 @@ register_msgring_handler(int major,
 	  mtx_unlock_spin(&msgrng_lock);
 
 	if (xlr_test_and_set(&msgring_int_enabled)) {
-		platform_prep_smp_launch();
-
+		create_msgring_thread(0);
 		cpu_establish_hardintr("msgring", (driver_filter_t *) msgring_process_fast_intr,
 			NULL, NULL, IRQ_MSGRING, 
 			INTR_TYPE_NET | INTR_FAST, &cookie);
-
-		/* configure the msgring interrupt on cpu 0 */
-		enable_msgring_int(NULL);
 	}
 	return 0;
 }
@@ -303,7 +403,8 @@ pic_init(void)
 		 * Use local scheduling and high polarity for all IRTs
 		 * Invalidate all IRTs, by default
 		 */
-		xlr_write_reg(mmio, PIC_IRT_1_BASE + i, (level << 30) | (1 << 6) | (PIC_IRQ_BASE + i));
+		xlr_write_reg(mmio, PIC_IRT_1_BASE + i, (level << 30) | (1 << 6) |
+		    (PIC_IRQ_BASE + i));
 	}
 	dbg_msg("PIC init now done\n");
 }
@@ -311,8 +412,6 @@ pic_init(void)
 void 
 on_chip_init(void)
 {
-	int i = 0, j = 0;
-
 	/* Set xlr_io_base to the run time value */
 	mtx_init(&msgrng_lock, "msgring", NULL, MTX_SPIN | MTX_RECURSE);
 	mtx_init(&xlr_pic_lock, "pic", NULL, MTX_SPIN);
@@ -325,8 +424,18 @@ on_chip_init(void)
 	pic_init();
 
 	xlr_msgring_cpu_init();
+}
+
+static void
+start_msgring_threads(void *arg)
+{
+	uint32_t cpu_mask;
+	int cpu;
 
-	for (i = 0; i < MAXCPU; i++)
-		for (j = 0; j < XLR_MAX_COUNTERS; j++)
-			atomic_set_int(&xlr_counters[i][j], 0);
+	cpu_mask = PCPU_GET(cpumask) | PCPU_GET(other_cpus);
+	for (cpu = 4; cpu < MAXCPU; cpu += 4)
+		if (cpu_mask & (1<<cpu))
+			create_msgring_thread(cpu);
 }
+
+SYSINIT(start_msgring_threads, SI_SUB_SMP, SI_ORDER_MIDDLE, start_msgring_threads, NULL);
diff --git a/sys/mips/rmi/pic.h b/sys/mips/rmi/pic.h
index fd7ffc6..05d2741 100644
--- a/sys/mips/rmi/pic.h
+++ b/sys/mips/rmi/pic.h
@@ -281,4 +281,16 @@ pic_delayed_ack(int irq, int haslock)
 	}
 }
 
+static inline
+void pic_send_ipi(int cpu, int ipi, int haslock)
+{
+        xlr_reg_t *mmio = xlr_io_mmio(XLR_IO_PIC_OFFSET);
+        int tid, pid;
+
+        tid = cpu & 0x3;
+        pid = (cpu >> 2) & 0x7;
+
+	xlr_write_reg(mmio, PIC_IPI,  (pid << 20) | (tid << 16) | ipi);
+}
+
 #endif				/* _RMI_PIC_H_ */
diff --git a/sys/mips/rmi/xlr_machdep.c b/sys/mips/rmi/xlr_machdep.c
index f66c1b0..a01d1de0 100644
--- a/sys/mips/rmi/xlr_machdep.c
+++ b/sys/mips/rmi/xlr_machdep.c
@@ -36,19 +36,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/rtprio.h>
 #include <sys/systm.h>
 #include <sys/interrupt.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/ktr.h>
 #include <sys/limits.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
-#include <sys/proc.h>
 #include <sys/random.h>
-#include <sys/resourcevar.h>
-#include <sys/sched.h>
-#include <sys/sysctl.h>
-#include <sys/unistd.h>
 
 #include <sys/cons.h>		/* cinit() */
 #include <sys/kdb.h>
@@ -87,16 +79,14 @@ __FBSDID("$FreeBSD$");
 #include <mips/rmi/perfmon.h>
 #endif
 
-
-
-void platform_prep_smp_launch(void);
+void mpwait(void);
+void enable_msgring_int(void *arg);
 
 unsigned long xlr_io_base = (unsigned long)(DEFAULT_XLR_IO_BASE);
 
 /* 4KB static data aread to keep a copy of the bootload env until
    the dynamic kenv is setup */
 char boot1_env[4096];
-extern unsigned long _gp;
 int rmi_spin_mutex_safe=0;
 /*
  * Parameters from boot loader
@@ -108,16 +98,6 @@ int xlr_argc;
 char **xlr_argv, **xlr_envp;
 uint64_t cpu_mask_info;
 uint32_t xlr_online_cpumask;
-
-#ifdef SMP
-static unsigned long xlr_secondary_gp[MAXCPU];
-static unsigned long xlr_secondary_sp[MAXCPU];
-
-#endif
-extern int mips_cpu_online_mask;
-extern int mips_cpu_logical_mask;
-uint32_t cpu_ltop_map[MAXCPU];
-uint32_t cpu_ptol_map[MAXCPU];
 uint32_t xlr_core_cpu_mask = 0x1;	/* Core 0 thread 0 is always there */
 
 void
@@ -130,27 +110,6 @@ platform_reset(void)
 	mmio[8] = 0x1;
 }
 
-void 
-platform_secondary_init(void)
-{
-#ifdef SMP
-	xlr_msgring_cpu_init();
-
-	/* Setup interrupts for secondary CPUs here */
-	mips_mask_hard_irq(IPI_SMP_CALL_FUNCTION);
-	mips_mask_hard_irq(IPI_STOP);
-	mips_mask_hard_irq(IPI_RENDEZVOUS);
-	mips_mask_hard_irq(IPI_AST);
-	mips_mask_hard_irq(IRQ_TIMER);
-#ifdef XLR_PERFMON
-	mips_mask_hard_irq(IPI_PERFMON);
-#endif
-
-	return;
-#endif
-}
-
-
 int xlr_asid_pcpu = 256;	/* This the default */
 int xlr_shtlb_enabled = 0;
 
@@ -168,13 +127,13 @@ setup_tlb_resource(void)
 	int mmu_setup;
 	int value = 0;
 	uint32_t cpu_map = xlr_boot1_info.cpu_online_map;
-	uint32_t thr_mask = cpu_map >> (xlr_cpu_id() << 2);
+	uint32_t thr_mask = cpu_map >> (xlr_core_id() << 2);
 	uint8_t core0 = xlr_boot1_info.cpu_online_map & 0xf;
 	uint8_t core_thr_mask;
 	int i = 0, count = 0;
 
 	/* If CPU0 did not enable shared TLB, other cores need to follow */
-	if ((xlr_cpu_id() != 0) && (xlr_shtlb_enabled == 0))
+	if ((xlr_core_id() != 0) && (xlr_shtlb_enabled == 0))
 		return;
 	/* First check if each core is brought up with the same mask */
 	for (i = 1; i < 8; i++) {
@@ -212,12 +171,14 @@ setup_tlb_resource(void)
 	mmu_setup |= (value << 1);
 
 	/* turn on global mode */
+#ifndef SMP
 	mmu_setup |= 0x01;
-
+#endif
 	write_32bit_phnx_ctrl_reg(4, 0, mmu_setup);
 
 }
 
+
 /*
  * Platform specific register setup for CPUs
  * XLR has control registers accessible with MFCR/MTCR instructions, this
@@ -233,6 +194,10 @@ platform_cpu_init()
 	uint32_t reg, val;
 	int thr_id = xlr_thr_id();
 
+/*
+ * XXX: SMP now need different wired mappings for threads 
+ * we cannot share TLBs.
+ */
 	if (thr_id == 0) {
 		if ((hw_env = getenv("xlr.shtlb")) != NULL) {
 			start = hw_env;
@@ -287,22 +252,6 @@ err_return:
 }
 
 
-#ifdef SMP
-extern void xlr_secondary_start(unsigned long, unsigned long, unsigned long);
-static void 
-xlr_secondary_entry(void *data)
-{
-	unsigned long sp, gp;
-	unsigned int cpu = (xlr_cpu_id() << 2) + xlr_thr_id();
-
-	sp = xlr_secondary_sp[cpu];
-	gp = xlr_secondary_gp[cpu];
-
-	xlr_secondary_start((unsigned long)mips_secondary_wait, sp, gp);
-}
-
-#endif
-
 static void 
 xlr_set_boot_flags(void)
 {
@@ -341,7 +290,6 @@ xlr_set_boot_flags(void)
 }
 extern uint32_t _end;
 
-
 static void
 mips_init(void)
 {
@@ -353,9 +301,6 @@ mips_init(void)
 	mips_cpu_init();
 	pmap_bootstrap();
 #ifdef DDB
-#ifdef SMP
-	setup_nmi();
-#endif				/* SMP */
 	kdb_init();
 	if (boothowto & RB_KDB) {
 		kdb_enter("Boot flags requested debugger", NULL);
@@ -521,29 +466,17 @@ platform_start(__register_t a0 __unused,
 	for (i = 1, j = 1; i < 32; i++) {
 		/* Allocate stack for all other cpus from fbsd kseg0 memory. */
 		if ((1U << i) & xlr_boot1_info.cpu_online_map) {
-			xlr_secondary_gp[i] =
-			    pmap_steal_memory(PAGE_SIZE);
-			if (!xlr_secondary_gp[i])
-				panic("Allocation failed for secondary cpu stacks");
-			xlr_secondary_sp[i] =
-			    xlr_secondary_gp[i] + PAGE_SIZE - CALLFRAME_SIZ;
-			xlr_secondary_gp[i] = (unsigned long)&_gp;
-			/* Build ltop and ptol cpu map. */
-			cpu_ltop_map[j] = i;
-			cpu_ptol_map[i] = j;
 			if ((i & 0x3) == 0)	/* store thread0 of each core */
 				xlr_core_cpu_mask |= (1 << j);
-			mips_cpu_logical_mask |= (1 << j);
 			j++;
 		}
 	}
 
-	mips_cpu_online_mask |= xlr_boot1_info.cpu_online_map;
 	wakeup = ((void (*) (void *, void *, unsigned int))
 	    (unsigned long)(xlr_boot1_info.wakeup));
 	printf("Waking up CPUs 0x%llx.\n", xlr_boot1_info.cpu_online_map & ~(0x1U));
 	if (xlr_boot1_info.cpu_online_map & ~(0x1U))
-		wakeup(xlr_secondary_entry, 0,
+		wakeup(mpwait, 0,
 		    (unsigned int)xlr_boot1_info.cpu_online_map);
 #endif
 
@@ -582,144 +515,59 @@ platform_trap_exit(void)
 {
 }
 
+#ifdef SMP
+int xlr_ap_release[MAXCPU];
 
-/*
- void
- platform_update_intrmask(int intr)
- {
-   write_c0_eimr64(read_c0_eimr64() | (1ULL<<intr));
- }
-*/
-
-void 
-disable_msgring_int(void *arg);
-void 
-enable_msgring_int(void *arg);
-void xlr_msgring_handler(struct trapframe *tf);
-int msgring_process_fast_intr(void *arg);
-
-struct msgring_ithread {
-	struct thread *i_thread;
-	u_int i_pending;
-	u_int i_flags;
-	int i_cpu;
-};
-struct msgring_ithread msgring_ithreads[MAXCPU];
-char ithd_name[MAXCPU][32];
-
-int
-msgring_process_fast_intr(void *arg)
+int platform_start_ap(int cpuid)
 {
-	int cpu = PCPU_GET(cpuid);
-	volatile struct msgring_ithread *it;
-	struct thread *td;
-
-	/* wakeup an appropriate intr_thread for processing this interrupt */
-	it = (volatile struct msgring_ithread *)&msgring_ithreads[cpu];
-	td = it->i_thread;
-
 	/*
-	 * Interrupt thread will enable the interrupts after processing all
-	 * messages
+	 * other cpus are enabled by the boot loader and they will be 
+	 * already looping in mpwait, release them
 	 */
-	disable_msgring_int(NULL);
-	atomic_store_rel_int(&it->i_pending, 1);
-	thread_lock(td);
-	if (TD_AWAITING_INTR(td)) {
-		TD_CLR_IWAIT(td);
-		sched_add(td, SRQ_INTR);
-	}
-	thread_unlock(td);
-	return FILTER_HANDLED;
+	atomic_store_rel_int(&xlr_ap_release[cpuid], 1);
+	return 0;
 }
 
-static void
-msgring_process(void *arg)
+void platform_init_ap(int processor_id)
 {
-	volatile struct msgring_ithread *ithd;
-	struct thread *td;
-	struct proc *p;
+	uint32_t stat;
 
-	td = curthread;
-	p = td->td_proc;
-	ithd = (volatile struct msgring_ithread *)arg;
-	KASSERT(ithd->i_thread == td,
-	    ("%s:msg_ithread and proc linkage out of sync", __func__));
+	/* Setup interrupts for secondary CPUs here */
+	stat = mips_rd_status();
+	stat |= MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT;
+	mips_wr_status(stat);
+
+	xlr_unmask_hard_irq((void *)platform_ipi_intrnum());
+	xlr_unmask_hard_irq((void *)IRQ_TIMER);
+	if (xlr_thr_id() == 0) {
+		xlr_msgring_cpu_init(); 
+	 	xlr_unmask_hard_irq((void *)IRQ_MSGRING);
+	}
 
-	/* First bind this thread to the right CPU */
-	thread_lock(td);
-	sched_bind(td, ithd->i_cpu);
-	thread_unlock(td);
+	return;
+}
 
-	//printf("Started %s on CPU %d\n", __FUNCTION__, ithd->i_cpu);
+int platform_ipi_intrnum(void) 
+{
+	return IRQ_IPI;
+}
 
-	while (1) {
-		while (ithd->i_pending) {
-			/*
-			 * This might need a full read and write barrier to
-			 * make sure that this write posts before any of the
-			 * memory or device accesses in the handlers.
-			 */
-			xlr_msgring_handler(NULL);
-			atomic_store_rel_int(&ithd->i_pending, 0);
-			enable_msgring_int(NULL);
-		}
-		if (!ithd->i_pending) {
-			thread_lock(td);
-			if (ithd->i_pending) {
-			  thread_unlock(td);
-			  continue;
-			}
-			sched_class(td, PRI_ITHD);
-			TD_SET_IWAIT(td);
-			mi_switch(SW_VOL, NULL);
-			thread_unlock(td);
-		}
-	}
+void platform_ipi_send(int cpuid)
+{
+	pic_send_ipi(cpuid, platform_ipi_intrnum(), 0);
+}
 
+void platform_ipi_clear(void)
+{
 }
-void 
-platform_prep_smp_launch(void)
+
+int platform_processor_id(void)
 {
-	int cpu;
-	uint32_t cpu_mask;
-	struct msgring_ithread *ithd;
-	struct thread *td;
-	struct proc *p;
-	int error;
-
-	cpu_mask = PCPU_GET(cpumask) | PCPU_GET(other_cpus);
-
-	/* Create kernel threads for message ring interrupt processing */
-	/* Currently create one task for thread 0 of each core */
-	for (cpu = 0; cpu < MAXCPU; cpu += 1) {
-
-		if (!((1 << cpu) & cpu_mask))
-			continue;
-
-		if ((cpu_ltop_map[cpu] % 4) != 0)
-			continue;
-
-		ithd = &msgring_ithreads[cpu];
-		sprintf(ithd_name[cpu], "msg_intr%d", cpu);
-		error = kproc_create(msgring_process,
-		    (void *)ithd,
-		    &p,
-		    (RFSTOPPED | RFHIGHPID),
-		    2,
-		    ithd_name[cpu]);
-
-		if (error)
-			panic("kproc_create() failed with %d", error);
-		td = FIRST_THREAD_IN_PROC(p);	/* XXXKSE */
-
-		thread_lock(td);
-		sched_class(td, PRI_ITHD);
-		TD_SET_IWAIT(td);
-		thread_unlock(td);
-		ithd->i_thread = td;
-		ithd->i_pending = 0;
-		ithd->i_cpu = cpu;
-		CTR2(KTR_INTR, "%s: created %s", __func__, ithd_name[cpu]);
-	}
+	return xlr_cpu_id();
 }
+
+int platform_num_processors(void)
+{
+	return fls(xlr_boot1_info.cpu_online_map);
+}
+#endif
diff --git a/sys/mips/rmi/xlr_pci.c b/sys/mips/rmi/xlr_pci.c
index 5f52e9e..314276f 100644
--- a/sys/mips/rmi/xlr_pci.c
+++ b/sys/mips/rmi/xlr_pci.c
@@ -636,4 +636,4 @@ static driver_t xlr_pcib_driver = {
 	sizeof(struct xlr_pcib_softc),
 };
 
-DRIVER_MODULE(pcib, nexus, xlr_pcib_driver, pcib_devclass, 0, 0);
+DRIVER_MODULE(pcib, iodi, xlr_pcib_driver, pcib_devclass, 0, 0);
diff --git a/sys/mips/rmi/xlrconfig.h b/sys/mips/rmi/xlrconfig.h
index 3ba96eb..c2aee99 100644
--- a/sys/mips/rmi/xlrconfig.h
+++ b/sys/mips/rmi/xlrconfig.h
@@ -25,6 +25,7 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+  __FBSDID("$FreeBSD$");
  *
  * RMI_BSD */
 #ifndef XLRCONFIG_H
@@ -127,39 +128,35 @@
         :"$8", "$9");                                           \
    } while(0)
 
-#if 0
-#define xlr_processor_id()                                      \
+#define xlr_cpu_id()                                            \
 ({int __id;                                                     \
  __asm__ __volatile__ (                                         \
            ".set push\n"                                        \
            ".set noreorder\n"                                   \
-           ".word 0x40088007\n"                                 \
-           "srl  $8, $8, 10\n"                                  \
-           "andi %0, $8, 0x3f\n"                                \
+           "mfc0 $8, $15, 1\n"                                  \
+           "andi %0, $8, 0x1f\n"                                \
            ".set pop\n"                                         \
            : "=r" (__id) : : "$8");                             \
  __id;})
-#endif
 
-#define xlr_cpu_id()                                        \
+#define xlr_core_id()                                           \
 ({int __id;                                                     \
  __asm__ __volatile__ (                                         \
            ".set push\n"                                        \
            ".set noreorder\n"                                   \
-           ".word 0x40088007\n"                                 \
-           "srl  $8, $8, 4\n"                                   \
-           "andi %0, $8, 0x7\n"                                \
+           "mfc0 $8, $15, 1\n"                                  \
+           "andi %0, $8, 0x1f\n"                                \
            ".set pop\n"                                         \
            : "=r" (__id) : : "$8");                             \
- __id;})
+ __id/4;})
 
-#define xlr_thr_id()                                        \
+#define xlr_thr_id()                                            \
 ({int __id;                                                     \
  __asm__ __volatile__ (                                         \
            ".set push\n"                                        \
            ".set noreorder\n"                                   \
-           ".word 0x40088007\n"                                 \
-           "andi %0, $8, 0x03\n"                                \
+           "mfc0 $8, $15, 1\n"                                  \
+           "andi %0, $8, 0x3\n"                                 \
            ".set pop\n"                                         \
            : "=r" (__id) : : "$8");                             \
  __id;})
@@ -333,4 +330,26 @@ xlr_mtcr(uint32_t reg, uint32_t val)
 	    :       "$8", "$9");
 }
 
+static __inline__ uint32_t
+xlr_paddr_lw(uint64_t paddr)
+{
+        uint32_t high, low, tmp;
+
+        high = 0x98000000 | (paddr >> 32);
+        low = paddr & 0xffffffff;
+
+        __asm__ __volatile__(
+                    ".set push         \n\t"
+                    ".set mips64       \n\t"
+                    "dsll32 %1, %1, 0  \n\t"
+                    "dsll32 %2, %2, 0  \n\t"  /* get rid of the */
+                    "dsrl32 %2, %2, 0  \n\t"  /* sign extend */
+                    "or     %1, %1, %2 \n\t"
+                    "lw     %0, 0(%1)  \n\t"
+                    ".set pop           \n"
+            :       "=r"(tmp)
+            :       "r"(high), "r"(low));
+
+	return tmp;
+}
 #endif