Add an abstraction layer to the 64-bit AIM MMU's page table manipulation

logic to support modifying the page table through a hypervisor. This
uses KOBJ inheritance to provide subclasses of the base 64-bit AIM MMU
class with additional methods for page table manipulation.

Many thanks to Peter Grehan for suggesting this design and implementing
the MMU KOBJ inheritance mechanism.

7 files changed, 1004 insertions, 644 deletions

diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index 83f75c4..4856715 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -83,6 +83,8 @@ powerpc/aim/locore.S		optional	aim no-obj
 powerpc/aim/machdep.c		optional	aim
 powerpc/aim/mmu_oea.c		optional	aim powerpc
 powerpc/aim/mmu_oea64.c		optional	aim
+powerpc/aim/moea64_if.m		optional	aim
+powerpc/aim/moea64_native.c	optional	aim
 powerpc/aim/mp_cpudep.c		optional	aim smp
 powerpc/aim/nexus.c		optional	aim
 powerpc/aim/ofwmagic.S		optional	aim
diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index 7bd07e1..6f99081 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -161,24 +161,6 @@ __FBSDID("$FreeBSD$");
 #define	VSID_TO_SR(vsid)	((vsid) & 0xf)
 #define	VSID_TO_HASH(vsid)	(((vsid) >> 4) & 0xfffff)
 
-#define	PVO_PTEGIDX_MASK	0x007		/* which PTEG slot */
-#define	PVO_PTEGIDX_VALID	0x008		/* slot is valid */
-#define	PVO_WIRED		0x010		/* PVO entry is wired */
-#define	PVO_MANAGED		0x020		/* PVO entry is managed */
-#define	PVO_EXECUTABLE		0x040		/* PVO entry is executable */
-#define	PVO_BOOTSTRAP		0x080		/* PVO entry allocated during
-						   bootstrap */
-#define PVO_FAKE		0x100		/* fictitious phys page */
-#define	PVO_VADDR(pvo)		((pvo)->pvo_vaddr & ~ADDR_POFF)
-#define	PVO_ISEXECUTABLE(pvo)	((pvo)->pvo_vaddr & PVO_EXECUTABLE)
-#define PVO_ISFAKE(pvo)		((pvo)->pvo_vaddr & PVO_FAKE)
-#define	PVO_PTEGIDX_GET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
-#define	PVO_PTEGIDX_ISSET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
-#define	PVO_PTEGIDX_CLR(pvo)	\
-	((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
-#define	PVO_PTEGIDX_SET(pvo, i)	\
-	((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
-
 #define	MOEA_PVO_CHECK(pvo)
 
 struct ofw_map {
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 506676f..cd791f8 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -155,79 +155,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/trap.h>
 #include <machine/mmuvar.h>
 
+#include "mmu_oea64.h"
 #include "mmu_if.h"
+#include "moea64_if.h"
 
-#define	MOEA_DEBUG
-
-#define TODO	panic("%s: not implemented", __func__);
 void moea64_release_vsid(uint64_t vsid);
 uintptr_t moea64_get_unique_vsid(void); 
 
-static __inline register_t
-cntlzd(volatile register_t a) {
-	register_t b;
-	__asm ("cntlzd %0, %1" : "=r"(b) : "r"(a));
-	return b;
-}
-
-#define	PTESYNC()	__asm __volatile("ptesync");
-#define	TLBSYNC()	__asm __volatile("tlbsync; ptesync");
-#define	SYNC()		__asm __volatile("sync");
-#define	EIEIO()		__asm __volatile("eieio");
-
-/*
- * The tlbie instruction must be executed in 64-bit mode
- * so we have to twiddle MSR[SF] around every invocation.
- * Just to add to the fun, exceptions must be off as well
- * so that we can't trap in 64-bit mode. What a pain.
- */
-struct mtx	tlbie_mutex;
-
-static __inline void
-TLBIE(uint64_t vpn) {
-#ifndef __powerpc64__
-	register_t vpn_hi, vpn_lo;
-	register_t msr;
-	register_t scratch;
-#endif
-
-	vpn <<= ADDR_PIDX_SHFT;
-	vpn &= ~(0xffffULL << 48);
-
-	mtx_lock_spin(&tlbie_mutex);
-#ifdef __powerpc64__
-	__asm __volatile("\
-	    ptesync; \
-	    tlbie %0; \
-	    eieio; \
-	    tlbsync; \
-	    ptesync;" 
-	:: "r"(vpn) : "memory");
-#else
-	vpn_hi = (uint32_t)(vpn >> 32);
-	vpn_lo = (uint32_t)vpn;
-
-	__asm __volatile("\
-	    mfmsr %0; \
-	    mr %1, %0; \
-	    insrdi %1,%5,1,0; \
-	    mtmsrd %1; isync; \
-	    ptesync; \
-	    \
-	    sld %1,%2,%4; \
-	    or %1,%1,%3; \
-	    tlbie %1; \
-	    \
-	    mtmsrd %0; isync; \
-	    eieio; \
-	    tlbsync; \
-	    ptesync;" 
-	: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
-	    : "memory");
-#endif
-	mtx_unlock_spin(&tlbie_mutex);
-}
-
 #define DISABLE_TRANS(msr)	msr = mfmsr(); mtmsr(msr & ~PSL_DR); isync()
 #define ENABLE_TRANS(msr)	mtmsr(msr); isync()
 
@@ -235,24 +169,6 @@ TLBIE(uint64_t vpn) {
 #define	VSID_TO_HASH(vsid)	(((vsid) >> 4) & 0xfffff)
 #define	VSID_HASH_MASK		0x0000007fffffffffULL
 
-#define	PVO_PTEGIDX_MASK	0x007UL		/* which PTEG slot */
-#define	PVO_PTEGIDX_VALID	0x008UL		/* slot is valid */
-#define	PVO_WIRED		0x010UL		/* PVO entry is wired */
-#define	PVO_MANAGED		0x020UL		/* PVO entry is managed */
-#define	PVO_BOOTSTRAP		0x080UL		/* PVO entry allocated during
-						   bootstrap */
-#define PVO_FAKE		0x100UL		/* fictitious phys page */
-#define PVO_LARGE		0x200UL		/* large page */
-#define	PVO_VADDR(pvo)		((pvo)->pvo_vaddr & ~ADDR_POFF)
-#define PVO_ISFAKE(pvo)		((pvo)->pvo_vaddr & PVO_FAKE)
-#define	PVO_PTEGIDX_GET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
-#define	PVO_PTEGIDX_ISSET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
-#define	PVO_PTEGIDX_CLR(pvo)	\
-	((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
-#define	PVO_PTEGIDX_SET(pvo, i)	\
-	((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
-#define	PVO_VSID(pvo)		((pvo)->pvo_vpn >> 16)
-
 #define	MOEA_PVO_CHECK(pvo)
 
 #define LOCK_TABLE() mtx_lock(&moea64_table_mutex)
@@ -277,7 +193,6 @@ static int	regions_sz, pregions_sz;
 
 extern void bs_remap_earlyboot(void);
 
-
 /*
  * Lock for the pteg and pvo tables.
  */
@@ -287,7 +202,6 @@ struct mtx	moea64_slb_mutex;
 /*
  * PTEG data.
  */
-static struct	lpteg *moea64_pteg_table;
 u_int		moea64_pteg_count;
 u_int		moea64_pteg_mask;
 
@@ -337,8 +251,8 @@ SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD,
     &moea64_pvo_remove_calls, 0, "");
 
 vm_offset_t	moea64_scratchpage_va[2];
-uint64_t	moea64_scratchpage_vpn[2];
-struct	lpte 	*moea64_scratchpage_pte[2];
+struct pvo_entry *moea64_scratchpage_pvo[2];
+uintptr_t	moea64_scratchpage_pte[2];
 struct	mtx	moea64_scratchpage_mtx;
 
 uint64_t 	moea64_large_page_mask = 0;
@@ -346,41 +260,23 @@ int		moea64_large_page_size = 0;
 int		moea64_large_page_shift = 0;
 
 /*
- * Allocate physical memory for use in moea64_bootstrap.
- */
-static vm_offset_t	moea64_bootstrap_alloc(vm_size_t, u_int);
-
-/*
- * PTE calls.
- */
-static int		moea64_pte_insert(u_int, struct lpte *);
-
-/*
  * PVO calls.
  */
-static int	moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *,
+static int	moea64_pvo_enter(mmu_t, pmap_t, uma_zone_t, struct pvo_head *,
 		    vm_offset_t, vm_offset_t, uint64_t, int);
-static void	moea64_pvo_remove(struct pvo_entry *);
+static void	moea64_pvo_remove(mmu_t, struct pvo_entry *);
 static struct	pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t);
-static struct	lpte *moea64_pvo_to_pte(const struct pvo_entry *);
 
 /*
  * Utility routines.
  */
-static void		moea64_bootstrap(mmu_t mmup, 
-			    vm_offset_t kernelstart, vm_offset_t kernelend);
-static void		moea64_cpu_bootstrap(mmu_t, int ap);
-static void		moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t,
-			    vm_prot_t, boolean_t);
-static boolean_t	moea64_query_bit(vm_page_t, u_int64_t);
-static u_int		moea64_clear_bit(vm_page_t, u_int64_t);
+static void		moea64_enter_locked(mmu_t, pmap_t, vm_offset_t,
+			    vm_page_t, vm_prot_t, boolean_t);
+static boolean_t	moea64_query_bit(mmu_t, vm_page_t, u_int64_t);
+static u_int		moea64_clear_bit(mmu_t, vm_page_t, u_int64_t);
 static void		moea64_kremove(mmu_t, vm_offset_t);
-static void		moea64_syncicache(pmap_t pmap, vm_offset_t va, 
+static void		moea64_syncicache(mmu_t, pmap_t pmap, vm_offset_t va, 
 			    vm_offset_t pa, vm_size_t sz);
-static void		tlbia(void);
-#ifdef __powerpc64__
-static void		slbia(void);
-#endif
 
 /*
  * Kernel MMU interface
@@ -463,8 +359,6 @@ static mmu_method_t moea64_methods[] = {
 	MMUMETHOD(mmu_page_set_memattr,	moea64_page_set_memattr),
 
 	/* Internal interfaces */
-	MMUMETHOD(mmu_bootstrap,       	moea64_bootstrap),
-	MMUMETHOD(mmu_cpu_bootstrap,   	moea64_cpu_bootstrap),
 	MMUMETHOD(mmu_mapdev,		moea64_mapdev),
 	MMUMETHOD(mmu_mapdev_attr,	moea64_mapdev_attr),
 	MMUMETHOD(mmu_unmapdev,		moea64_unmapdev),
@@ -476,7 +370,7 @@ static mmu_method_t moea64_methods[] = {
 	{ 0, 0 }
 };
 
-MMU_DEF(oea64_mmu, MMU_TYPE_G5, moea64_methods, 0);
+MMU_DEF(oea64_mmu, "mmu_oea64_base", moea64_methods, 0);
 
 static __inline u_int
 va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
@@ -542,81 +436,6 @@ moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
 	pt->pte_lo = pte_lo;
 }
 
-static __inline void
-moea64_pte_synch(struct lpte *pt, struct lpte *pvo_pt)
-{
-
-	ASSERT_TABLE_LOCK();
-
-	pvo_pt->pte_lo |= pt->pte_lo & (LPTE_REF | LPTE_CHG);
-}
-
-static __inline void
-moea64_pte_clear(struct lpte *pt, uint64_t vpn, u_int64_t ptebit)
-{
-	ASSERT_TABLE_LOCK();
-
-	/*
-	 * As shown in Section 7.6.3.2.3
-	 */
-	pt->pte_lo &= ~ptebit;
-	TLBIE(vpn);
-}
-
-static __inline void
-moea64_pte_set(struct lpte *pt, struct lpte *pvo_pt)
-{
-
-	ASSERT_TABLE_LOCK();
-	pvo_pt->pte_hi |= LPTE_VALID;
-
-	/*
-	 * Update the PTE as defined in section 7.6.3.1.
-	 * Note that the REF/CHG bits are from pvo_pt and thus should have
-	 * been saved so this routine can restore them (if desired).
-	 */
-	pt->pte_lo = pvo_pt->pte_lo;
-	EIEIO();
-	pt->pte_hi = pvo_pt->pte_hi;
-	PTESYNC();
-	moea64_pte_valid++;
-}
-
-static __inline void
-moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn)
-{
-	ASSERT_TABLE_LOCK();
-	pvo_pt->pte_hi &= ~LPTE_VALID;
-
-	/*
-	 * Force the reg & chg bits back into the PTEs.
-	 */
-	SYNC();
-
-	/*
-	 * Invalidate the pte.
-	 */
-	pt->pte_hi &= ~LPTE_VALID;
-	TLBIE(vpn);
-
-	/*
-	 * Save the reg & chg bits.
-	 */
-	moea64_pte_synch(pt, pvo_pt);
-	moea64_pte_valid--;
-}
-
-static __inline void
-moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn)
-{
-
-	/*
-	 * Invalidate the PTE
-	 */
-	moea64_pte_unset(pt, pvo_pt, vpn);
-	moea64_pte_set(pt, pvo_pt);
-}
-
 static __inline uint64_t
 moea64_calc_wimg(vm_offset_t pa, vm_memattr_t ma)
 {
@@ -696,49 +515,6 @@ om_cmp(const void *a, const void *b)
 }
 
 static void
-moea64_cpu_bootstrap(mmu_t mmup, int ap)
-{
-	int i = 0;
-	#ifdef __powerpc64__
-	struct slb *slb = PCPU_GET(slb);
-	#endif
-
-	/*
-	 * Initialize segment registers and MMU
-	 */
-
-	mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync();
-
-	/*
-	 * Install kernel SLB entries
-	 */
-
-	#ifdef __powerpc64__
-		slbia();
-
-		for (i = 0; i < 64; i++) {
-			if (!(slb[i].slbe & SLBE_VALID))
-				continue;
-
-			__asm __volatile ("slbmte %0, %1" :: 
-			    "r"(slb[i].slbv), "r"(slb[i].slbe)); 
-		}
-	#else
-		for (i = 0; i < 16; i++)
-			mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
-	#endif
-
-	/*
-	 * Install page table
-	 */
-
-	__asm __volatile ("ptesync; mtsdr1 %0; isync"
-	    :: "r"((uintptr_t)moea64_pteg_table 
-		     | (64 - cntlzd(moea64_pteg_mask >> 11))));
-	tlbia();
-}
-
-static void
 moea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz)
 {
 	struct ofw_map	translations[sz/sizeof(struct ofw_map)];
@@ -874,7 +650,7 @@ moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart,
 			    pregions[i].mr_start + pregions[i].mr_size)
 				pte_lo |= LPTE_G;
 
-			moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+			moea64_pvo_enter(mmup, kernel_pmap, moea64_upvo_zone,
 				    &moea64_pvo_kunmanaged, pa, pa,
 				    pte_lo, PVO_WIRED | PVO_LARGE |
 				    VM_PROT_EXECUTE);
@@ -882,10 +658,6 @@ moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart,
 		}
 		PMAP_UNLOCK(kernel_pmap);
 	} else {
-		size = moea64_pteg_count * sizeof(struct lpteg);
-		off = (vm_offset_t)(moea64_pteg_table);
-		for (pa = off; pa < off + size; pa += PAGE_SIZE) 
-			moea64_kenter(mmup, pa, pa);
 		size = sizeof(struct pvo_head) * moea64_pteg_count;
 		off = (vm_offset_t)(moea64_pvo_table);
 		for (pa = off; pa < off + size; pa += PAGE_SIZE) 
@@ -911,18 +683,11 @@ moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart,
 	ENABLE_TRANS(msr);
 }
 
-static void
-moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+void
+moea64_early_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 {
-	ihandle_t	mmui;
-	phandle_t	chosen;
-	phandle_t	mmu;
-	size_t		sz;
 	int		i, j;
-	vm_size_t	size, physsz, hwphyssz;
-	vm_offset_t	pa, va;
-	register_t	msr;
-	void		*dpcpu;
+	vm_size_t	physsz, hwphyssz;
 
 #ifndef __powerpc64__
 	/* We don't have a direct map since there is no BAT */
@@ -1009,9 +774,6 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 
 	physmem = btoc(physsz);
 
-	/*
-	 * Allocate PTEG table.
-	 */
 #ifdef PTEGCOUNT
 	moea64_pteg_count = PTEGCOUNT;
 #else
@@ -1022,27 +784,20 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 
 	moea64_pteg_count >>= 1;
 #endif /* PTEGCOUNT */
+}
 
-	size = moea64_pteg_count * sizeof(struct lpteg);
-	CTR2(KTR_PMAP, "moea64_bootstrap: %d PTEGs, %d bytes", 
-	    moea64_pteg_count, size);
+void
+moea64_mid_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+{
+	vm_size_t	size;
+	register_t	msr;
+	int		i;
 
 	/*
-	 * We now need to allocate memory. This memory, to be allocated,
-	 * has to reside in a page table. The page table we are about to
-	 * allocate. We don't have BAT. So drop to data real mode for a minute
-	 * as a measure of last resort. We do this a couple times.
+	 * Set PTEG mask
 	 */
-
-	moea64_pteg_table = (struct lpteg *)moea64_bootstrap_alloc(size, size);
-	DISABLE_TRANS(msr);
-	bzero((void *)moea64_pteg_table, moea64_pteg_count * sizeof(struct lpteg));
-	ENABLE_TRANS(msr);
-
 	moea64_pteg_mask = moea64_pteg_count - 1;
 
-	CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table);
-
 	/*
 	 * Allocate pv/overflow lists.
 	 */
@@ -1066,11 +821,6 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	mtx_init(&moea64_slb_mutex, "SLB table", NULL, MTX_DEF);
 
 	/*
-	 * Initialize the TLBIE lock. TLBIE can only be executed by one CPU.
-	 */
-	mtx_init(&tlbie_mutex, "tlbie mutex", NULL, MTX_SPIN);
-
-	/*
 	 * Initialise the unmanaged pvo pool.
 	 */
 	moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc(
@@ -1109,6 +859,18 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	 */
 
 	moea64_setup_direct_map(mmup, kernelstart, kernelend);
+}
+
+void
+moea64_late_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+{
+	ihandle_t	mmui;
+	phandle_t	chosen;
+	phandle_t	mmu;
+	size_t		sz;
+	int		i;
+	vm_offset_t	pa, va;
+	void		*dpcpu;
 
 	/*
 	 * Set up the Open Firmware pmap and add its mappings if not in real
@@ -1137,7 +899,7 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	/*
 	 * Initialize MMU and remap early physical mappings
 	 */
-	moea64_cpu_bootstrap(mmup,0);
+	MMU_CPU_BOOTSTRAP(mmup,0);
 	mtmsr(mfmsr() | PSL_DR | PSL_IR); isync();
 	pmap_bootstrapped++;
 	bs_remap_earlyboot();
@@ -1173,47 +935,6 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	#endif
 
 	/*
-	 * Allocate some things for page zeroing. We put this directly
-	 * in the page table, marked with LPTE_LOCKED, to avoid any
-	 * of the PVO book-keeping or other parts of the VM system
-	 * from even knowing that this hack exists.
-	 */
-
-	if (!hw_direct_map) {
-		mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL,
-		    MTX_DEF);
-		for (i = 0; i < 2; i++) {
-			struct lpte pt;
-			uint64_t vsid;
-			int pteidx, ptegidx;
-
-			moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
-			virtual_end -= PAGE_SIZE;
-
-			LOCK_TABLE();
-			
-			vsid = va_to_vsid(kernel_pmap,
-			    moea64_scratchpage_va[i]);
-			moea64_pte_create(&pt, vsid, moea64_scratchpage_va[i],
-			    LPTE_NOEXEC, 0);
-			pt.pte_hi |= LPTE_LOCKED;
-
-			moea64_scratchpage_vpn[i] = (vsid << 16) |
-			    ((moea64_scratchpage_va[i] & ADDR_PIDX) >>
-			    ADDR_PIDX_SHFT);
-			ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i], 0);
-			pteidx = moea64_pte_insert(ptegidx, &pt);
-			if (pt.pte_hi & LPTE_HID)
-				ptegidx ^= moea64_pteg_mask;
-
-			moea64_scratchpage_pte[i] =
-			    &moea64_pteg_table[ptegidx].pt[pteidx];
-
-			UNLOCK_TABLE();
-		}
-	}
-
-	/*
 	 * Allocate a kernel stack with a guard page for thread0 and map it
 	 * into the kernel page map.
 	 */
@@ -1255,6 +976,36 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 		va += PAGE_SIZE;
 	}
 	dpcpu_init(dpcpu, 0);
+
+	/*
+	 * Allocate some things for page zeroing. We put this directly
+	 * in the page table, marked with LPTE_LOCKED, to avoid any
+	 * of the PVO book-keeping or other parts of the VM system
+	 * from even knowing that this hack exists.
+	 */
+
+	if (!hw_direct_map) {
+		mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL,
+		    MTX_DEF);
+		for (i = 0; i < 2; i++) {
+			moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
+			virtual_end -= PAGE_SIZE;
+
+			moea64_kenter(mmup, moea64_scratchpage_va[i], 0);
+
+			moea64_scratchpage_pvo[i] = moea64_pvo_find_va(
+			    kernel_pmap, (vm_offset_t)moea64_scratchpage_va[i]);
+			LOCK_TABLE();
+			moea64_scratchpage_pte[i] = MOEA64_PVO_TO_PTE(
+			    mmup, moea64_scratchpage_pvo[i]);
+			moea64_scratchpage_pvo[i]->pvo_pte.lpte.pte_hi
+			    |= LPTE_LOCKED;
+			MOEA64_PTE_CHANGE(mmup, moea64_scratchpage_pte[i],
+			    &moea64_scratchpage_pvo[i]->pvo_pte.lpte,
+			    moea64_scratchpage_pvo[i]->pvo_vpn);
+			UNLOCK_TABLE();
+		}
+	}
 }
 
 /*
@@ -1294,7 +1045,7 @@ void
 moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
 {
 	struct	pvo_entry *pvo;
-	struct	lpte *pt;
+	uintptr_t pt;
 	uint64_t vsid;
 	int	i, ptegidx;
 
@@ -1303,7 +1054,7 @@ moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
 
 	if (pvo != NULL) {
 		LOCK_TABLE();
-		pt = moea64_pvo_to_pte(pvo);
+		pt = MOEA64_PVO_TO_PTE(mmu, pvo);
 
 		if (wired) {
 			if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
@@ -1317,9 +1068,9 @@ moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
 			pvo->pvo_pte.lpte.pte_hi &= ~LPTE_WIRED;
 		}
 
-		if (pt != NULL) {
+		if (pt != -1) {
 			/* Update wiring flag in page table. */
-			moea64_pte_change(pt, &pvo->pvo_pte.lpte,
+			MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
 			    pvo->pvo_vpn);
 		} else if (wired) {
 			/*
@@ -1330,7 +1081,8 @@ moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
 			ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
 			    pvo->pvo_vaddr & PVO_LARGE);
 
-			i = moea64_pte_insert(ptegidx, &pvo->pvo_pte.lpte);
+			i = MOEA64_PTE_INSERT(mmu, ptegidx, &pvo->pvo_pte.lpte);
+			
 			if (i >= 0) {
 				PVO_PTEGIDX_CLR(pvo);
 				PVO_PTEGIDX_SET(pvo, i);
@@ -1350,22 +1102,18 @@ moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
  */
 
 static __inline
-void moea64_set_scratchpage_pa(int which, vm_offset_t pa) {
+void moea64_set_scratchpage_pa(mmu_t mmup, int which, vm_offset_t pa) {
 
 	KASSERT(!hw_direct_map, ("Using OEA64 scratchpage with a direct map!"));
 	mtx_assert(&moea64_scratchpage_mtx, MA_OWNED);
 
-	moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID;
-	TLBIE(moea64_scratchpage_vpn[which]);
-	
-	moea64_scratchpage_pte[which]->pte_lo &= 
+	moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo &=
 	    ~(LPTE_WIMG | LPTE_RPGN);
-	moea64_scratchpage_pte[which]->pte_lo |=
+	moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo |=
 	    moea64_calc_wimg(pa, VM_MEMATTR_DEFAULT) | (uint64_t)pa;
-	EIEIO();
-
-	moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID;
-	PTESYNC(); isync();
+	MOEA64_PTE_CHANGE(mmup, moea64_scratchpage_pte[which],
+	    &moea64_scratchpage_pvo[which]->pvo_pte.lpte,
+	    moea64_scratchpage_pvo[which]->pvo_vpn);
 }
 
 void
@@ -1382,8 +1130,8 @@ moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst)
 	} else {
 		mtx_lock(&moea64_scratchpage_mtx);
 
-		moea64_set_scratchpage_pa(0,src);
-		moea64_set_scratchpage_pa(1,dst);
+		moea64_set_scratchpage_pa(mmu, 0, src);
+		moea64_set_scratchpage_pa(mmu, 1, dst);
 
 		kcopy((void *)moea64_scratchpage_va[0], 
 		    (void *)moea64_scratchpage_va[1], PAGE_SIZE);
@@ -1406,7 +1154,7 @@ moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
 		bzero((caddr_t)pa + off, size);
 	} else {
 		mtx_lock(&moea64_scratchpage_mtx);
-		moea64_set_scratchpage_pa(0,pa);
+		moea64_set_scratchpage_pa(mmu, 0, pa);
 		bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
 		mtx_unlock(&moea64_scratchpage_mtx);
 	}
@@ -1427,7 +1175,7 @@ moea64_zero_page(mmu_t mmu, vm_page_t m)
 	if (!hw_direct_map) {
 		mtx_lock(&moea64_scratchpage_mtx);
 
-		moea64_set_scratchpage_pa(0,pa);
+		moea64_set_scratchpage_pa(mmu, 0, pa);
 		va = moea64_scratchpage_va[0];
 	} else {
 		va = pa;
@@ -1459,7 +1207,7 @@ moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
 
 	vm_page_lock_queues();
 	PMAP_LOCK(pmap);
-	moea64_enter_locked(pmap, va, m, prot, wired);
+	moea64_enter_locked(mmu, pmap, va, m, prot, wired);
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
@@ -1473,8 +1221,8 @@ moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
  */
 
 static void
-moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
-    boolean_t wired)
+moea64_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
+    vm_prot_t prot, boolean_t wired)
 {
 	struct		pvo_head *pvo_head;
 	uma_zone_t	zone;
@@ -1528,20 +1276,20 @@ moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 	if ((m->flags & PG_FICTITIOUS) != 0)
 		pvo_flags |= PVO_FAKE;
 
-	error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),
-	    pte_lo, pvo_flags);
+	error = moea64_pvo_enter(mmu, pmap, zone, pvo_head, va,
+	    VM_PAGE_TO_PHYS(m), pte_lo, pvo_flags);
 
 	/*
 	 * Flush the page from the instruction cache if this page is
 	 * mapped executable and cacheable.
 	 */
-	if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
-		moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
-	}
+	if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
+		moea64_syncicache(mmu, pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
 }
 
 static void
-moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz)
+moea64_syncicache(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_offset_t pa,
+    vm_size_t sz)
 {
 
 	/*
@@ -1568,7 +1316,7 @@ moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz)
 
 		mtx_lock(&moea64_scratchpage_mtx);
 
-		moea64_set_scratchpage_pa(1,pa & ~ADDR_POFF);
+		moea64_set_scratchpage_pa(mmu, 1, pa & ~ADDR_POFF);
 		__syncicache((void *)(moea64_scratchpage_va[1] + 
 		    (va & ADDR_POFF)), sz);
 
@@ -1600,7 +1348,7 @@ moea64_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end,
 	vm_page_lock_queues();
 	PMAP_LOCK(pm);
 	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
-		moea64_enter_locked(pm, start + ptoa(diff), m, prot &
+		moea64_enter_locked(mmu, pm, start + ptoa(diff), m, prot &
 		    (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
 		m = TAILQ_NEXT(m, listq);
 	}
@@ -1615,8 +1363,8 @@ moea64_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m,
 
 	vm_page_lock_queues();
 	PMAP_LOCK(pm);
-	moea64_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE),
-	    FALSE);
+	moea64_enter_locked(mmu, pm, va, m,
+	    prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pm);
 }
@@ -1669,6 +1417,8 @@ retry:
 	return (m);
 }
 
+static mmu_t installed_mmu;
+
 static void *
 moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 
 {
@@ -1709,7 +1459,7 @@ moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 
 	va = VM_PAGE_TO_PHYS(m);
 
-	moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+	moea64_pvo_enter(installed_mmu, kernel_pmap, moea64_upvo_zone,
 	    &moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M,
 	    PVO_WIRED | PVO_BOOTSTRAP);
 
@@ -1736,6 +1486,7 @@ moea64_init(mmu_t mmu)
 	    UMA_ZONE_VM | UMA_ZONE_NOFREE);
 
 	if (!hw_direct_map) {
+		installed_mmu = mmu;
 		uma_zone_set_allocf(moea64_upvo_zone,moea64_uma_page_alloc);
 		uma_zone_set_allocf(moea64_mpvo_zone,moea64_uma_page_alloc);
 	}
@@ -1749,7 +1500,7 @@ moea64_is_referenced(mmu_t mmu, vm_page_t m)
 
 	KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
 	    ("moea64_is_referenced: page %p is not managed", m));
-	return (moea64_query_bit(m, PTE_REF));
+	return (moea64_query_bit(mmu, m, PTE_REF));
 }
 
 boolean_t
@@ -1768,7 +1519,7 @@ moea64_is_modified(mmu_t mmu, vm_page_t m)
 	if ((m->oflags & VPO_BUSY) == 0 &&
 	    (m->flags & PG_WRITEABLE) == 0)
 		return (FALSE);
-	return (moea64_query_bit(m, LPTE_CHG));
+	return (moea64_query_bit(mmu, m, LPTE_CHG));
 }
 
 boolean_t
@@ -1790,7 +1541,7 @@ moea64_clear_reference(mmu_t mmu, vm_page_t m)
 
 	KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
 	    ("moea64_clear_reference: page %p is not managed", m));
-	moea64_clear_bit(m, LPTE_REF);
+	moea64_clear_bit(mmu, m, LPTE_REF);
 }
 
 void
@@ -1810,7 +1561,7 @@ moea64_clear_modify(mmu_t mmu, vm_page_t m)
 	 */
 	if ((m->flags & PG_WRITEABLE) == 0)
 		return;
-	moea64_clear_bit(m, LPTE_CHG);
+	moea64_clear_bit(mmu, m, LPTE_CHG);
 }
 
 /*
@@ -1820,7 +1571,7 @@ void
 moea64_remove_write(mmu_t mmu, vm_page_t m)
 {
 	struct	pvo_entry *pvo;
-	struct	lpte *pt;
+	uintptr_t pt;
 	pmap_t	pmap;
 	uint64_t lo;
 
@@ -1838,21 +1589,21 @@ moea64_remove_write(mmu_t mmu, vm_page_t m)
 		return;
 	vm_page_lock_queues();
 	lo = moea64_attr_fetch(m);
-	SYNC();
+	powerpc_sync();
 	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
 		pmap = pvo->pvo_pmap;
 		PMAP_LOCK(pmap);
 		LOCK_TABLE();
 		if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
-			pt = moea64_pvo_to_pte(pvo);
+			pt = MOEA64_PVO_TO_PTE(mmu, pvo);
 			pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
 			pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
-			if (pt != NULL) {
-				moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+			if (pt != -1) {
+				MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte);
 				lo |= pvo->pvo_pte.lpte.pte_lo;
 				pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
-				moea64_pte_change(pt, &pvo->pvo_pte.lpte,
-				    pvo->pvo_vpn);
+				MOEA64_PTE_CHANGE(mmu, pt,
+				    &pvo->pvo_pte.lpte, pvo->pvo_vpn);
 				if (pvo->pvo_pmap == kernel_pmap)
 					isync();
 			}
@@ -1886,7 +1637,7 @@ moea64_ts_referenced(mmu_t mmu, vm_page_t m)
 
 	KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
 	    ("moea64_ts_referenced: page %p is not managed", m));
-	return (moea64_clear_bit(m, LPTE_REF));
+	return (moea64_clear_bit(mmu, m, LPTE_REF));
 }
 
 /*
@@ -1897,7 +1648,7 @@ moea64_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma)
 {
 	struct	pvo_entry *pvo;
 	struct  pvo_head *pvo_head;
-	struct	lpte *pt;
+	uintptr_t pt;
 	pmap_t	pmap;
 	uint64_t lo;
 
@@ -1913,11 +1664,11 @@ moea64_page_set_memattr(mmu_t mmu, vm_page_t m, vm_memattr_t ma)
 		pmap = pvo->pvo_pmap;
 		PMAP_LOCK(pmap);
 		LOCK_TABLE();
-		pt = moea64_pvo_to_pte(pvo);
+		pt = MOEA64_PVO_TO_PTE(mmu, pvo);
 		pvo->pvo_pte.lpte.pte_lo &= ~LPTE_WIMG;
 		pvo->pvo_pte.lpte.pte_lo |= lo;
-		if (pt != NULL) {
-			moea64_pte_change(pt, &pvo->pvo_pte.lpte,
+		if (pt != -1) {
+			MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
 			    pvo->pvo_vpn);
 			if (pvo->pvo_pmap == kernel_pmap)
 				isync();
@@ -1941,7 +1692,7 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma)
 	pte_lo = moea64_calc_wimg(pa, ma);
 
 	PMAP_LOCK(kernel_pmap);
-	error = moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+	error = moea64_pvo_enter(mmu, kernel_pmap, moea64_upvo_zone,
 	    &moea64_pvo_kunmanaged, va, pa, pte_lo, 
 	    PVO_WIRED | VM_PROT_EXECUTE);
 
@@ -1952,9 +1703,8 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma)
 	/*
 	 * Flush the memory from the instruction cache.
 	 */
-	if ((pte_lo & (LPTE_I | LPTE_G)) == 0) {
+	if ((pte_lo & (LPTE_I | LPTE_G)) == 0)
 		__syncicache((void *)va, PAGE_SIZE);
-	}
 	PMAP_UNLOCK(kernel_pmap);
 }
 
@@ -2183,7 +1933,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
     vm_prot_t prot)
 {
 	struct	pvo_entry *pvo;
-	struct	lpte *pt;
+	uintptr_t pt;
 
 	CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva,
 	    eva, prot);
@@ -2209,7 +1959,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
 		 * copy.
 		 */
 		LOCK_TABLE();
-		pt = moea64_pvo_to_pte(pvo);
+		pt = MOEA64_PVO_TO_PTE(mmu, pvo);
 
 		/*
 		 * Change the protection of the page.
@@ -2223,11 +1973,12 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
 		/*
 		 * If the PVO is in the page table, update that pte as well.
 		 */
-		if (pt != NULL) {
-			moea64_pte_change(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
+		if (pt != -1) {
+			MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
+			    pvo->pvo_vpn);
 			if ((pvo->pvo_pte.lpte.pte_lo & 
 			    (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
-				moea64_syncicache(pm, sva,
+				moea64_syncicache(mmu, pm, sva,
 				    pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN,
 				    PAGE_SIZE);
 			}
@@ -2314,7 +2065,7 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
 	for (; sva < eva; sva += PAGE_SIZE) {
 		pvo = moea64_pvo_find_va(pm, sva);
 		if (pvo != NULL)
-			moea64_pvo_remove(pvo);
+			moea64_pvo_remove(mmu, pvo);
 	}
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pm);
@@ -2339,7 +2090,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m)
 		MOEA_PVO_CHECK(pvo);	/* sanity check */
 		pmap = pvo->pvo_pmap;
 		PMAP_LOCK(pmap);
-		moea64_pvo_remove(pvo);
+		moea64_pvo_remove(mmu, pvo);
 		PMAP_UNLOCK(pmap);
 	}
 	if ((m->flags & PG_WRITEABLE) && moea64_is_modified(mmu, m)) {
@@ -2355,7 +2106,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m)
  * Can only be called from moea64_bootstrap before avail start and end are
  * calculated.
  */
-static vm_offset_t
+vm_offset_t
 moea64_bootstrap_alloc(vm_size_t size, u_int align)
 {
 	vm_offset_t	s, e;
@@ -2396,53 +2147,10 @@ moea64_bootstrap_alloc(vm_size_t size, u_int align)
 	panic("moea64_bootstrap_alloc: could not allocate memory");
 }
 
-static void
-tlbia(void)
-{
-	vm_offset_t i;
-	#ifndef __powerpc64__
-	register_t msr, scratch;
-	#endif
-
-	TLBSYNC();
-
-	for (i = 0; i < 0xFF000; i += 0x00001000) {
-		#ifdef __powerpc64__
-		__asm __volatile("tlbiel %0" :: "r"(i));
-		#else
-		__asm __volatile("\
-		    mfmsr %0; \
-		    mr %1, %0; \
-		    insrdi %1,%3,1,0; \
-		    mtmsrd %1; \
-		    isync; \
-		    \
-		    tlbiel %2; \
-		    \
-		    mtmsrd %0; \
-		    isync;" 
-		: "=r"(msr), "=r"(scratch) : "r"(i), "r"(1));
-		#endif
-	}
-
-	EIEIO();
-	TLBSYNC();
-}
-
-#ifdef __powerpc64__
-static void
-slbia(void)
-{
-	register_t seg0;
-
-	__asm __volatile ("slbia");
-	__asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
-}
-#endif
-
 static int
-moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
-    vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags)
+moea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone,
+    struct pvo_head *pvo_head, vm_offset_t va, vm_offset_t pa,
+    uint64_t pte_lo, int flags)
 {
 	struct	 pvo_entry *pvo;
 	uint64_t vsid;
@@ -2488,7 +2196,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
 			    (pte_lo & LPTE_PP)) {
 			    	if (!(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) {
 					/* Re-insert if spilled */
-					i = moea64_pte_insert(ptegidx,
+					i = MOEA64_PTE_INSERT(mmu, ptegidx,
 					    &pvo->pvo_pte.lpte);
 					if (i >= 0)
 						PVO_PTEGIDX_SET(pvo, i);
@@ -2497,7 +2205,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
 				UNLOCK_TABLE();
 				return (0);
 			}
-			moea64_pvo_remove(pvo);
+			moea64_pvo_remove(mmu, pvo);
 			break;
 		}
 	}
@@ -2572,7 +2280,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
 	/*
 	 * We hope this succeeds but it isn't required.
 	 */
-	i = moea64_pte_insert(ptegidx, &pvo->pvo_pte.lpte);
+	i = MOEA64_PTE_INSERT(mmu, ptegidx, &pvo->pvo_pte.lpte);
 	if (i >= 0) {
 		PVO_PTEGIDX_SET(pvo, i);
 	} else {
@@ -2598,18 +2306,18 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
 }
 
 static void
-moea64_pvo_remove(struct pvo_entry *pvo)
+moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo)
 {
-	struct	lpte *pt;
+	uintptr_t pt;
 
 	/*
 	 * If there is an active pte entry, we need to deactivate it (and
 	 * save the ref & cfg bits).
 	 */
 	LOCK_TABLE();
-	pt = moea64_pvo_to_pte(pvo);
-	if (pt != NULL) {
-		moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
+	pt = MOEA64_PVO_TO_PTE(mmu, pvo);
+	if (pt != -1) {
+		MOEA64_PTE_UNSET(mmu, pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
 		PVO_PTEGIDX_CLR(pvo);
 	} else {
 		moea64_pte_overflow--;
@@ -2698,192 +2406,11 @@ moea64_pvo_find_va(pmap_t pm, vm_offset_t va)
 	return (pvo);
 }
 
-static struct lpte *
-moea64_pvo_to_pte(const struct pvo_entry *pvo)
-{
-	struct lpte 	*pt;
-	int		pteidx, ptegidx;
-	uint64_t	vsid;
-
-	ASSERT_TABLE_LOCK();
-
-	/* If the PTEG index is not set, then there is no page table entry */
-	if (!PVO_PTEGIDX_ISSET(pvo))
-		return (NULL);
-
-	/*
-	 * Calculate the ptegidx
-	 */
-	vsid = PVO_VSID(pvo);
-	ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
-	    pvo->pvo_vaddr & PVO_LARGE);
-
-	/*
-	 * We can find the actual pte entry without searching by grabbing
-	 * the PTEG index from 3 unused bits in pvo_vaddr and by
-	 * noticing the HID bit.
-	 */
-	if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
-		ptegidx ^= moea64_pteg_mask;
-
-	pteidx = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo);
-
-	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && 
-	    !PVO_PTEGIDX_ISSET(pvo)) {
-		panic("moea64_pvo_to_pte: pvo %p has valid pte in pvo but no "
-		    "valid pte index", pvo);
-	}
-
-	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0 && 
-	    PVO_PTEGIDX_ISSET(pvo)) {
-		panic("moea64_pvo_to_pte: pvo %p has valid pte index in pvo "
-		    "pvo but no valid pte", pvo);
-	}
-
-	pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
-	if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) == 
-	    LPTE_VALID) {
-		if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) {
-			panic("moea64_pvo_to_pte: pvo %p has valid pte in "
-			    "moea64_pteg_table %p but invalid in pvo", pvo, pt);
-		}
-
-		if (((pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo) & 
-		    ~(LPTE_M|LPTE_CHG|LPTE_REF)) != 0) {
-			panic("moea64_pvo_to_pte: pvo %p pte does not match "
-			    "pte %p in moea64_pteg_table difference is %#x", 
-			    pvo, pt,
-			    (uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo));
-		}
-
-		return (pt);
-	}
-
-	if (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) {
-		panic("moea64_pvo_to_pte: pvo %p has invalid pte %p in "
-		    "moea64_pteg_table but valid in pvo", pvo, pt);
-	}
-
-	return (NULL);
-}
-
-static __inline int
-moea64_pte_spillable_ident(u_int ptegidx)
-{
-	struct	lpte *pt;
-	int	i, j, k;
-
-	/* Start at a random slot */
-	i = mftb() % 8;
-	k = -1;
-	for (j = 0; j < 8; j++) {
-		pt = &moea64_pteg_table[ptegidx].pt[(i + j) % 8];
-		if (pt->pte_hi & (LPTE_LOCKED | LPTE_WIRED))
-			continue;
-
-		/* This is a candidate, so remember it */
-		k = (i + j) % 8;
-
-		/* Try to get a page that has not been used lately */
-		if (!(pt->pte_lo & LPTE_REF))
-			return (k);
-	}
-	
-	return (k);
-}
-
-static int
-moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
-{
-	struct	lpte *pt;
-	struct	pvo_entry *pvo;
-	u_int	pteg_bktidx;
-	int	i;
-
-	ASSERT_TABLE_LOCK();
-
-	/*
-	 * First try primary hash.
-	 */
-	pteg_bktidx = ptegidx;
-	for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
-		if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
-			pvo_pt->pte_hi &= ~LPTE_HID;
-			moea64_pte_set(pt, pvo_pt);
-			return (i);
-		}
-	}
-
-	/*
-	 * Now try secondary hash.
-	 */
-	pteg_bktidx ^= moea64_pteg_mask;
-	for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
-		if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
-			pvo_pt->pte_hi |= LPTE_HID;
-			moea64_pte_set(pt, pvo_pt);
-			return (i);
-		}
-	}
-
-	/*
-	 * Out of luck. Find a PTE to sacrifice.
-	 */
-	pteg_bktidx = ptegidx;
-	i = moea64_pte_spillable_ident(pteg_bktidx);
-	if (i < 0) {
-		pteg_bktidx ^= moea64_pteg_mask;
-		i = moea64_pte_spillable_ident(pteg_bktidx);
-	}
-
-	if (i < 0) {
-		/* No freeable slots in either PTEG? We're hosed. */
-		panic("moea64_pte_insert: overflow");
-		return (-1);
-	}
-
-	if (pteg_bktidx == ptegidx)
-		pvo_pt->pte_hi &= ~LPTE_HID;
-	else
-		pvo_pt->pte_hi |= LPTE_HID;
-
-	/*
-	 * Synchronize the sacrifice PTE with its PVO, then mark both
-	 * invalid. The PVO will be reused when/if the VM system comes
-	 * here after a fault.
-	 */
-	pt = &moea64_pteg_table[pteg_bktidx].pt[i];
-
-	if (pt->pte_hi & LPTE_HID)
-		pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
-
-	LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) {
-		if (pvo->pvo_pte.lpte.pte_hi == pt->pte_hi) {
-			KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 
-			    ("Invalid PVO for valid PTE!"));
-			moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
-			PVO_PTEGIDX_CLR(pvo);
-			moea64_pte_overflow++;
-			break;
-		}
-	}
-
-	KASSERT(pvo->pvo_pte.lpte.pte_hi == pt->pte_hi,
-	   ("Unable to find PVO for spilled PTE"));
-
-	/*
-	 * Set the new PTE.
-	 */
-	moea64_pte_set(pt, pvo_pt);
-
-	return (i);
-}
-
 static boolean_t
-moea64_query_bit(vm_page_t m, u_int64_t ptebit)
+moea64_query_bit(mmu_t mmu, vm_page_t m, u_int64_t ptebit)
 {
 	struct	pvo_entry *pvo;
-	struct	lpte *pt;
+	uintptr_t pt;
 
 	if (moea64_attr_fetch(m) & ptebit)
 		return (TRUE);
@@ -2910,7 +2437,7 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit)
 	 * themselves.  Sync so that any pending REF/CHG bits are flushed to
 	 * the PTEs.
 	 */
-	SYNC();
+	powerpc_sync();
 	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
 		MOEA_PVO_CHECK(pvo);	/* sanity check */
 
@@ -2920,9 +2447,9 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit)
 		 * ptebit is set, cache it and return success.
 		 */
 		LOCK_TABLE();
-		pt = moea64_pvo_to_pte(pvo);
-		if (pt != NULL) {
-			moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+		pt = MOEA64_PVO_TO_PTE(mmu, pvo);
+		if (pt != -1) {
+			MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte);
 			if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
 				UNLOCK_TABLE();
 
@@ -2940,11 +2467,11 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit)
 }
 
 static u_int
-moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
+moea64_clear_bit(mmu_t mmu, vm_page_t m, u_int64_t ptebit)
 {
 	u_int	count;
 	struct	pvo_entry *pvo;
-	struct	lpte *pt;
+	uintptr_t pt;
 
 	vm_page_lock_queues();
 
@@ -2960,7 +2487,7 @@ moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
 	 * table, we don't have to worry about further accesses setting the
 	 * REF/CHG bits.
 	 */
-	SYNC();
+	powerpc_sync();
 
 	/*
 	 * For each pvo entry, clear the pvo's ptebit.  If this pvo has a
@@ -2971,12 +2498,13 @@ moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
 		MOEA_PVO_CHECK(pvo);	/* sanity check */
 
 		LOCK_TABLE();
-		pt = moea64_pvo_to_pte(pvo);
-		if (pt != NULL) {
-			moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+		pt = MOEA64_PVO_TO_PTE(mmu, pvo);
+		if (pt != -1) {
+			MOEA64_PTE_SYNCH(mmu, pt, &pvo->pvo_pte.lpte);
 			if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
 				count++;
-				moea64_pte_clear(pt, pvo->pvo_vpn, ptebit);
+				MOEA64_PTE_CLEAR(mmu, pt, &pvo->pvo_pte.lpte,
+				    pvo->pvo_vpn, ptebit);
 			}
 		}
 		pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
@@ -3058,7 +2586,7 @@ moea64_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size)
 	kmem_free(kernel_map, base, size);
 }
 
-static void
+void
 moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz)
 {
 	struct pvo_entry *pvo;
@@ -3074,7 +2602,7 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz)
 		if (pvo != NULL) {
 			pa = (pvo->pvo_pte.pte.pte_lo & LPTE_RPGN) |
 			    (va & ADDR_POFF);
-			moea64_syncicache(pm, va, pa, len);
+			moea64_syncicache(mmu, pm, va, pa, len);
 		}
 		va += len;
 		sz -= len;
diff --git a/sys/powerpc/aim/mmu_oea64.h b/sys/powerpc/aim/mmu_oea64.h
new file mode 100644
index 0000000..101181d
--- /dev/null
+++ b/sys/powerpc/aim/mmu_oea64.h
@@ -0,0 +1,77 @@
+/*-
+ * Copyright (C) 2010 Nathan Whitehorn
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _POWERPC_AIM_MMU_OEA64_H
+#define _POWERPC_AIM_MMU_OEA64_H
+
+#include <machine/mmuvar.h>
+
+extern mmu_def_t oea64_mmu;
+
+/*
+ * Helper routines
+ */
+
+/* Allocate physical memory for use in moea64_bootstrap. */
+vm_offset_t	moea64_bootstrap_alloc(vm_size_t, u_int);
+
+/*
+ * Bootstrap subroutines
+ *
+ * An MMU_BOOTSTRAP() implementation looks like this:
+ *   moea64_early_bootstrap();
+ *   Allocate Page Table
+ *   moea64_mid_bootstrap();
+ *   Add mappings for MMU resources
+ *   moea64_late_bootstrap();
+ */
+
+void		moea64_early_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
+		    vm_offset_t kernelend);
+void		moea64_mid_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
+		    vm_offset_t kernelend);
+void		moea64_late_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
+		    vm_offset_t kernelend);
+
+/*
+ * Statistics
+ */
+
+extern u_int	moea64_pte_valid;
+extern u_int	moea64_pte_overflow;
+
+/*
+ * State variables
+ */
+
+extern struct pvo_head *moea64_pvo_table;
+extern int		moea64_large_page_shift;
+extern u_int		moea64_pteg_count;
+extern u_int		moea64_pteg_mask;
+
+#endif /* _POWERPC_AIM_MMU_OEA64_H */
+
diff --git a/sys/powerpc/aim/moea64_if.m b/sys/powerpc/aim/moea64_if.m
new file mode 100644
index 0000000..f041838
--- /dev/null
+++ b/sys/powerpc/aim/moea64_if.m
@@ -0,0 +1,115 @@
+#-
+# Copyright (c) 2010 Nathan Whitehorn
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/systm.h>
+        
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+
+#include <machine/mmuvar.h>
+
+/**
+ * MOEA64 kobj methods for 64-bit Book-S page table
+ * manipulation routines used, for example, by hypervisors.
+ */
+
+INTERFACE moea64;
+
+
+/**
+ * Copy ref/changed bits from PTE referenced by _pt_cookie to _pvo_pt.
+ */
+METHOD void pte_synch {
+	mmu_t		_mmu;
+	uintptr_t	_pt_cookie;
+	struct lpte	*_pvo_pt;
+};
+
+/**
+ * Clear bits ptebit (a mask) from the low word of the PTE referenced by
+ * _pt_cookie. Note that _pvo_pt is for reference use only -- the bit should
+ * NOT be cleared there.
+ */
+METHOD void pte_clear {
+	mmu_t		_mmu;
+	uintptr_t	_pt_cookie;
+	struct lpte	*_pvo_pt;
+	uint64_t	_vpn;
+	uint64_t	_ptebit;
+};
+
+/**
+ * Invalidate the PTE referenced by _pt_cookie, synchronizing its validity
+ * and ref/changed bits after completion.
+ */
+METHOD void pte_unset {
+	mmu_t		_mmu;
+	uintptr_t	_pt_cookie;
+	struct lpte	*_pvo_pt;
+	uint64_t	_vpn;
+};
+
+/**
+ * Update the PTE referenced by _pt_cookie with the values in _pvo_pt,
+ * making sure that the values of ref/changed bits are preserved and
+ * synchronized back to _pvo_pt.
+ */
+METHOD void pte_change {
+	mmu_t		_mmu;
+	uintptr_t	_pt_cookie;
+	struct lpte	*_pvo_pt;
+	uint64_t	_vpn;
+};
+	
+
+/**
+ * Insert the PTE _pvo_pt into the PTEG group _ptegidx, returning the index
+ * of the PTE in its group at completion, or -1 if no slots were free. Must
+ * not replace PTEs marked LPTE_WIRED or LPTE_LOCKED, and must set LPTE_HID
+ * and LPTE_VALID appropriately in _pvo_pt.
+ */
+METHOD int pte_insert {
+	mmu_t		_mmu;
+	u_int		_ptegidx;
+	struct lpte	*_pvo_pt;
+};
+
+/**
+ * Return the page table reference cookie corresponding to _pvo, or -1 if
+ * the _pvo is not currently in the page table.
+ */
+METHOD uintptr_t pvo_to_pte {
+	mmu_t		_mmu;
+	const struct pvo_entry *_pvo;
+};
+
+
diff --git a/sys/powerpc/aim/moea64_native.c b/sys/powerpc/aim/moea64_native.c
new file mode 100644
index 0000000..a386b93
--- /dev/null
+++ b/sys/powerpc/aim/moea64_native.c
@@ -0,0 +1,637 @@
+/*-
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $
+ */
+/*-
+ * Copyright (C) 2001 Benno Rice.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Native 64-bit page table operations for running without a hypervisor.
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <sys/kdb.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_pager.h>
+
+#include <machine/md_var.h>
+#include <machine/mmuvar.h>
+
+#include "mmu_oea64.h"
+#include "mmu_if.h"
+#include "moea64_if.h"
+
+#define	PTESYNC()	__asm __volatile("ptesync");
+#define	TLBSYNC()	__asm __volatile("tlbsync; ptesync");
+#define	SYNC()		__asm __volatile("sync");
+#define	EIEIO()		__asm __volatile("eieio");
+
+#define	VSID_HASH_MASK	0x0000007fffffffffULL
+
+/*
+ * The tlbie instruction must be executed in 64-bit mode
+ * so we have to twiddle MSR[SF] around every invocation.
+ * Just to add to the fun, exceptions must be off as well
+ * so that we can't trap in 64-bit mode. What a pain.
+ */
+struct mtx	tlbie_mutex;
+
+static __inline void
+TLBIE(uint64_t vpn) {
+#ifndef __powerpc64__
+	register_t vpn_hi, vpn_lo;
+	register_t msr;
+	register_t scratch;
+#endif
+
+	vpn <<= ADDR_PIDX_SHFT;
+	vpn &= ~(0xffffULL << 48);
+
+	mtx_lock_spin(&tlbie_mutex);
+#ifdef __powerpc64__
+	__asm __volatile("\
+	    ptesync; \
+	    tlbie %0; \
+	    eieio; \
+	    tlbsync; \
+	    ptesync;" 
+	:: "r"(vpn) : "memory");
+#else
+	vpn_hi = (uint32_t)(vpn >> 32);
+	vpn_lo = (uint32_t)vpn;
+
+	__asm __volatile("\
+	    mfmsr %0; \
+	    mr %1, %0; \
+	    insrdi %1,%5,1,0; \
+	    mtmsrd %1; isync; \
+	    ptesync; \
+	    \
+	    sld %1,%2,%4; \
+	    or %1,%1,%3; \
+	    tlbie %1; \
+	    \
+	    mtmsrd %0; isync; \
+	    eieio; \
+	    tlbsync; \
+	    ptesync;" 
+	: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
+	    : "memory");
+#endif
+	mtx_unlock_spin(&tlbie_mutex);
+}
+
+#define DISABLE_TRANS(msr)	msr = mfmsr(); mtmsr(msr & ~PSL_DR); isync()
+#define ENABLE_TRANS(msr)	mtmsr(msr); isync()
+
+/*
+ * PTEG data.
+ */
+static struct	lpteg *moea64_pteg_table;
+
+/*
+ * PTE calls.
+ */
+static int	moea64_pte_insert_native(mmu_t, u_int, struct lpte *);
+static uintptr_t moea64_pvo_to_pte_native(mmu_t, const struct pvo_entry *);
+static void	moea64_pte_synch_native(mmu_t, uintptr_t pt,
+		    struct lpte *pvo_pt);
+static void	moea64_pte_clear_native(mmu_t, uintptr_t pt,
+		    struct lpte *pvo_pt, uint64_t vpn, uint64_t ptebit);
+static void	moea64_pte_change_native(mmu_t, uintptr_t pt,
+		    struct lpte *pvo_pt, uint64_t vpn);
+static void	moea64_pte_unset_native(mmu_t mmu, uintptr_t pt,
+		    struct lpte *pvo_pt, uint64_t vpn);
+
+/*
+ * Utility routines.
+ */
+static void		moea64_bootstrap_native(mmu_t mmup, 
+			    vm_offset_t kernelstart, vm_offset_t kernelend);
+static void		moea64_cpu_bootstrap_native(mmu_t, int ap);
+static void		tlbia(void);
+
+static mmu_method_t moea64_native_methods[] = {
+	/* Internal interfaces */
+	MMUMETHOD(mmu_bootstrap,	moea64_bootstrap_native),
+	MMUMETHOD(mmu_cpu_bootstrap,	moea64_cpu_bootstrap_native),
+
+	MMUMETHOD(moea64_pte_synch,	moea64_pte_synch_native),
+	MMUMETHOD(moea64_pte_clear,	moea64_pte_clear_native),	
+	MMUMETHOD(moea64_pte_unset,	moea64_pte_unset_native),	
+	MMUMETHOD(moea64_pte_change,	moea64_pte_change_native),	
+	MMUMETHOD(moea64_pte_insert,	moea64_pte_insert_native),	
+	MMUMETHOD(moea64_pvo_to_pte,	moea64_pvo_to_pte_native),	
+
+	{ 0, 0 }
+};
+
+MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods,
+    0, oea64_mmu);
+
+static __inline u_int
+va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
+{
+	uint64_t hash;
+	int shift;
+
+	shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT;
+	hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >>
+	    shift);
+	return (hash & moea64_pteg_mask);
+}
+
+static void
+moea64_pte_synch_native(mmu_t mmu, uintptr_t pt_cookie, struct lpte *pvo_pt)
+{
+	struct lpte *pt = (struct lpte *)pt_cookie;
+
+	pvo_pt->pte_lo |= pt->pte_lo & (LPTE_REF | LPTE_CHG);
+}
+
+static void
+moea64_pte_clear_native(mmu_t mmu, uintptr_t pt_cookie, struct lpte *pvo_pt,
+    uint64_t vpn, uint64_t ptebit)
+{
+	struct lpte *pt = (struct lpte *)pt_cookie;
+
+	/*
+	 * As shown in Section 7.6.3.2.3
+	 */
+	pt->pte_lo &= ~ptebit;
+	TLBIE(vpn);
+}
+
+static void
+moea64_pte_set_native(struct lpte *pt, struct lpte *pvo_pt)
+{
+
+	pvo_pt->pte_hi |= LPTE_VALID;
+
+	/*
+	 * Update the PTE as defined in section 7.6.3.1.
+	 * Note that the REF/CHG bits are from pvo_pt and thus should have
+	 * been saved so this routine can restore them (if desired).
+	 */
+	pt->pte_lo = pvo_pt->pte_lo;
+	EIEIO();
+	pt->pte_hi = pvo_pt->pte_hi;
+	PTESYNC();
+	moea64_pte_valid++;
+}
+
+static void
+moea64_pte_unset_native(mmu_t mmu, uintptr_t pt_cookie, struct lpte *pvo_pt,
+    uint64_t vpn)
+{
+	struct lpte *pt = (struct lpte *)pt_cookie;
+
+	pvo_pt->pte_hi &= ~LPTE_VALID;
+
+	/*
+	 * Force the reg & chg bits back into the PTEs.
+	 */
+	SYNC();
+
+	/*
+	 * Invalidate the pte.
+	 */
+	pt->pte_hi &= ~LPTE_VALID;
+	TLBIE(vpn);
+
+	/*
+	 * Save the reg & chg bits.
+	 */
+	moea64_pte_synch_native(mmu, pt_cookie, pvo_pt);
+	moea64_pte_valid--;
+}
+
+static void
+moea64_pte_change_native(mmu_t mmu, uintptr_t pt, struct lpte *pvo_pt,
+    uint64_t vpn)
+{
+
+	/*
+	 * Invalidate the PTE
+	 */
+	moea64_pte_unset_native(mmu, pt, pvo_pt, vpn);
+	moea64_pte_set_native((struct lpte *)pt, pvo_pt);
+}
+
+static void
+moea64_cpu_bootstrap_native(mmu_t mmup, int ap)
+{
+	int i = 0;
+	#ifdef __powerpc64__
+	struct slb *slb = PCPU_GET(slb);
+	register_t seg0;
+	#endif
+
+	/*
+	 * Initialize segment registers and MMU
+	 */
+
+	mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync();
+
+	/*
+	 * Install kernel SLB entries
+	 */
+
+	#ifdef __powerpc64__
+		__asm __volatile ("slbia");
+		__asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) :
+		    "r"(0));
+
+		for (i = 0; i < 64; i++) {
+			if (!(slb[i].slbe & SLBE_VALID))
+				continue;
+
+			__asm __volatile ("slbmte %0, %1" :: 
+			    "r"(slb[i].slbv), "r"(slb[i].slbe)); 
+		}
+	#else
+		for (i = 0; i < 16; i++)
+			mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
+	#endif
+
+	/*
+	 * Install page table
+	 */
+
+	__asm __volatile ("ptesync; mtsdr1 %0; isync"
+	    :: "r"((uintptr_t)moea64_pteg_table 
+		     | (uintptr_t)(flsl(moea64_pteg_mask >> 11))));
+	tlbia();
+}
+
+static void
+moea64_bootstrap_native(mmu_t mmup, vm_offset_t kernelstart,
+    vm_offset_t kernelend)
+{
+	vm_size_t	size;
+	vm_offset_t	off;
+	vm_paddr_t	pa;
+	register_t	msr;
+
+	moea64_early_bootstrap(mmup, kernelstart, kernelend);
+
+	/*
+	 * Allocate PTEG table.
+	 */
+
+	size = moea64_pteg_count * sizeof(struct lpteg);
+	CTR2(KTR_PMAP, "moea64_bootstrap: %d PTEGs, %d bytes", 
+	    moea64_pteg_count, size);
+
+	/*
+	 * We now need to allocate memory. This memory, to be allocated,
+	 * has to reside in a page table. The page table we are about to
+	 * allocate. We don't have BAT. So drop to data real mode for a minute
+	 * as a measure of last resort. We do this a couple times.
+	 */
+
+	moea64_pteg_table = (struct lpteg *)moea64_bootstrap_alloc(size, size);
+	DISABLE_TRANS(msr);
+	bzero((void *)moea64_pteg_table, moea64_pteg_count * sizeof(struct lpteg));
+	ENABLE_TRANS(msr);
+
+	CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table);
+
+	/*
+	 * Initialize the TLBIE lock. TLBIE can only be executed by one CPU.
+	 */
+	mtx_init(&tlbie_mutex, "tlbie mutex", NULL, MTX_SPIN);
+
+	moea64_mid_bootstrap(mmup, kernelstart, kernelend);
+
+	/*
+	 * Add a mapping for the page table itself if there is no direct map.
+	 */
+	if (!hw_direct_map) {
+		size = moea64_pteg_count * sizeof(struct lpteg);
+		off = (vm_offset_t)(moea64_pteg_table);
+		DISABLE_TRANS(msr);
+		for (pa = off; pa < off + size; pa += PAGE_SIZE)
+			pmap_kenter(pa, pa);
+		ENABLE_TRANS(msr);
+	}
+
+	/* Bring up virtual memory */
+	moea64_late_bootstrap(mmup, kernelstart, kernelend);
+}
+
+static void
+tlbia(void)
+{
+	vm_offset_t i;
+	#ifndef __powerpc64__
+	register_t msr, scratch;
+	#endif
+
+	TLBSYNC();
+
+	for (i = 0; i < 0xFF000; i += 0x00001000) {
+		#ifdef __powerpc64__
+		__asm __volatile("tlbiel %0" :: "r"(i));
+		#else
+		__asm __volatile("\
+		    mfmsr %0; \
+		    mr %1, %0; \
+		    insrdi %1,%3,1,0; \
+		    mtmsrd %1; \
+		    isync; \
+		    \
+		    tlbiel %2; \
+		    \
+		    mtmsrd %0; \
+		    isync;" 
+		: "=r"(msr), "=r"(scratch) : "r"(i), "r"(1));
+		#endif
+	}
+
+	EIEIO();
+	TLBSYNC();
+}
+
+static uintptr_t
+moea64_pvo_to_pte_native(mmu_t mmu, const struct pvo_entry *pvo)
+{
+	struct lpte 	*pt;
+	int		pteidx, ptegidx;
+	uint64_t	vsid;
+
+	/* If the PTEG index is not set, then there is no page table entry */
+	if (!PVO_PTEGIDX_ISSET(pvo))
+		return (-1);
+
+	/*
+	 * Calculate the ptegidx
+	 */
+	vsid = PVO_VSID(pvo);
+	ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
+	    pvo->pvo_vaddr & PVO_LARGE);
+
+	/*
+	 * We can find the actual pte entry without searching by grabbing
+	 * the PTEG index from 3 unused bits in pvo_vaddr and by
+	 * noticing the HID bit.
+	 */
+	if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
+		ptegidx ^= moea64_pteg_mask;
+
+	pteidx = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo);
+
+	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && 
+	    !PVO_PTEGIDX_ISSET(pvo)) {
+		panic("moea64_pvo_to_pte: pvo %p has valid pte in pvo but no "
+		    "valid pte index", pvo);
+	}
+
+	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0 && 
+	    PVO_PTEGIDX_ISSET(pvo)) {
+		panic("moea64_pvo_to_pte: pvo %p has valid pte index in pvo "
+		    "pvo but no valid pte", pvo);
+	}
+
+	pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
+	if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) == 
+	    LPTE_VALID) {
+		if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) {
+			panic("moea64_pvo_to_pte: pvo %p has valid pte in "
+			    "moea64_pteg_table %p but invalid in pvo", pvo, pt);
+		}
+
+		if (((pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo) & 
+		    ~(LPTE_M|LPTE_CHG|LPTE_REF)) != 0) {
+			panic("moea64_pvo_to_pte: pvo %p pte does not match "
+			    "pte %p in moea64_pteg_table difference is %#x", 
+			    pvo, pt,
+			    (uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo));
+		}
+
+		return ((uintptr_t)pt);
+	}
+
+	if (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) {
+		panic("moea64_pvo_to_pte: pvo %p has invalid pte %p in "
+		    "moea64_pteg_table but valid in pvo", pvo, pt);
+	}
+
+	return (-1);
+}
+
+static __inline int
+moea64_pte_spillable_ident(u_int ptegidx)
+{
+	struct	lpte *pt;
+	int	i, j, k;
+
+	/* Start at a random slot */
+	i = mftb() % 8;
+	k = -1;
+	for (j = 0; j < 8; j++) {
+		pt = &moea64_pteg_table[ptegidx].pt[(i + j) % 8];
+		if (pt->pte_hi & (LPTE_LOCKED | LPTE_WIRED))
+			continue;
+
+		/* This is a candidate, so remember it */
+		k = (i + j) % 8;
+
+		/* Try to get a page that has not been used lately */
+		if (!(pt->pte_lo & LPTE_REF))
+			return (k);
+	}
+	
+	return (k);
+}
+
+static int
+moea64_pte_insert_native(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt)
+{
+	struct	lpte *pt;
+	struct	pvo_entry *pvo;
+	u_int	pteg_bktidx;
+	int	i;
+
+	/*
+	 * First try primary hash.
+	 */
+	pteg_bktidx = ptegidx;
+	for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
+		if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
+			pvo_pt->pte_hi &= ~LPTE_HID;
+			moea64_pte_set_native(pt, pvo_pt);
+			return (i);
+		}
+	}
+
+	/*
+	 * Now try secondary hash.
+	 */
+	pteg_bktidx ^= moea64_pteg_mask;
+	for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
+		if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
+			pvo_pt->pte_hi |= LPTE_HID;
+			moea64_pte_set_native(pt, pvo_pt);
+			return (i);
+		}
+	}
+
+	/*
+	 * Out of luck. Find a PTE to sacrifice.
+	 */
+	pteg_bktidx = ptegidx;
+	i = moea64_pte_spillable_ident(pteg_bktidx);
+	if (i < 0) {
+		pteg_bktidx ^= moea64_pteg_mask;
+		i = moea64_pte_spillable_ident(pteg_bktidx);
+	}
+
+	if (i < 0) {
+		/* No freeable slots in either PTEG? We're hosed. */
+		panic("moea64_pte_insert: overflow");
+		return (-1);
+	}
+
+	if (pteg_bktidx == ptegidx)
+		pvo_pt->pte_hi &= ~LPTE_HID;
+	else
+		pvo_pt->pte_hi |= LPTE_HID;
+
+	/*
+	 * Synchronize the sacrifice PTE with its PVO, then mark both
+	 * invalid. The PVO will be reused when/if the VM system comes
+	 * here after a fault.
+	 */
+	pt = &moea64_pteg_table[pteg_bktidx].pt[i];
+
+	if (pt->pte_hi & LPTE_HID)
+		pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
+
+	LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) {
+		if (pvo->pvo_pte.lpte.pte_hi == pt->pte_hi) {
+			KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 
+			    ("Invalid PVO for valid PTE!"));
+			moea64_pte_unset_native(mmu, (uintptr_t)pt,
+			    &pvo->pvo_pte.lpte, pvo->pvo_vpn);
+			PVO_PTEGIDX_CLR(pvo);
+			moea64_pte_overflow++;
+			break;
+		}
+	}
+
+	KASSERT(pvo->pvo_pte.lpte.pte_hi == pt->pte_hi,
+	   ("Unable to find PVO for spilled PTE"));
+
+	/*
+	 * Set the new PTE.
+	 */
+	moea64_pte_set_native(pt, pvo_pt);
+
+	return (i);
+}
+
diff --git a/sys/powerpc/include/pmap.h b/sys/powerpc/include/pmap.h
index c030416..2b26185 100644
--- a/sys/powerpc/include/pmap.h
+++ b/sys/powerpc/include/pmap.h
@@ -121,6 +121,25 @@ struct pvo_entry {
 };
 LIST_HEAD(pvo_head, pvo_entry);
 
+#define	PVO_PTEGIDX_MASK	0x007UL		/* which PTEG slot */
+#define	PVO_PTEGIDX_VALID	0x008UL		/* slot is valid */
+#define	PVO_WIRED		0x010UL		/* PVO entry is wired */
+#define	PVO_MANAGED		0x020UL		/* PVO entry is managed */
+#define	PVO_EXECUTABLE		0x040UL		/* PVO entry is executable */
+#define	PVO_BOOTSTRAP		0x080UL		/* PVO entry allocated during
+						   bootstrap */
+#define PVO_FAKE		0x100UL		/* fictitious phys page */
+#define PVO_LARGE		0x200UL		/* large page */
+#define	PVO_VADDR(pvo)		((pvo)->pvo_vaddr & ~ADDR_POFF)
+#define PVO_ISFAKE(pvo)		((pvo)->pvo_vaddr & PVO_FAKE)
+#define	PVO_PTEGIDX_GET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
+#define	PVO_PTEGIDX_ISSET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
+#define	PVO_PTEGIDX_CLR(pvo)	\
+	((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
+#define	PVO_PTEGIDX_SET(pvo, i)	\
+	((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
+#define	PVO_VSID(pvo)		((pvo)->pvo_vpn >> 16)
+
 struct	md_page {
 	u_int64_t	 mdpg_attrs;
 	vm_memattr_t	 mdpg_cache_attrs;