Add support for 64-bit PowerPC CPUs operating in the 64-bit bridge mode

provided, for example, on the PowerPC 970 (G5), as well as on related CPUs
like the POWER3 and POWER4.

This also adds support for various built-in hardware found on Apple G5
hardware (e.g. the IBM CPC925 northbridge).

Reviewed by:    grehan

24 files changed, 3742 insertions, 216 deletions

diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index da8f35f..75476f5 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -130,6 +130,8 @@ extern vm_offset_t ksym_start, ksym_end;
 
 int cold = 1;
 int cacheline_size = 32;
+int ppc64 = 0;
+int hw_direct_map = 1;
 
 struct pcpu __pcpu[MAXCPU];
 
@@ -230,10 +232,13 @@ cpu_startup(void *dummy)
 
 extern char	kernel_text[], _end[];
 
+extern void	*testppc64, *testppc64size;
+extern void	*restorebridge, *restorebridgesize;
+extern void	*rfid_patch, *rfi_patch1, *rfi_patch2;
 #ifdef SMP
 extern void	*rstcode, *rstsize;
 #endif
-extern void	*trapcode, *trapsize;
+extern void	*trapcode, *trapcode64, *trapsize;
 extern void	*alitrap, *alisize;
 extern void	*dsitrap, *dsisize;
 extern void	*decrint, *decrsize;
@@ -245,11 +250,16 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
 {
 	struct		pcpu *pc;
 	vm_offset_t	end;
+	void		*generictrap;
+	size_t		trap_offset;
 	void		*kmdp;
         char		*env;
+	int		vers;
+	uint32_t	msr, scratch;
 
 	end = 0;
 	kmdp = NULL;
+	trap_offset = 0;
 
 	/*
 	 * Parse metadata if present and fetch parameters.  Must be done
@@ -315,6 +325,26 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
 		printf("powerpc_init: no loader metadata.\n");
 	}
 
+	/*
+	 * Set cacheline_size based on the CPU model.
+	 */
+
+	vers = mfpvr() >> 16;
+	switch (vers) {
+		case IBM970:
+		case IBM970FX:
+		case IBM970MP:
+		case IBM970GX:
+			cacheline_size = 128;
+			break;
+		default:
+			cacheline_size = 32;
+	}
+
+	/*
+	 * Init KDB
+	 */
+
 	kdb_init();
 
 	/*
@@ -322,47 +352,110 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
 	 *      Disable translation in case the vector area
 	 *      hasn't been mapped (G5)
 	 */
-	mtmsr(mfmsr() & ~(PSL_IR | PSL_DR));
+	msr = mfmsr();
+	mtmsr(msr & ~(PSL_IR | PSL_DR));
 	isync();
+
+	/*
+	 * Figure out whether we need to use the 64 bit PMAP. This works by
+	 * executing an instruction that is only legal on 64-bit PPC (mtmsrd),
+	 * and setting ppc64 = 0 if that causes a trap.
+	 */
+
+	ppc64 = 1;
+
+	bcopy(&testppc64, (void *)EXC_PGM,  (size_t)&testppc64size);
+	__syncicache((void *)EXC_PGM, (size_t)&testppc64size);
+
+	__asm __volatile("\
+		mfmsr %0;	\
+		mtsprg2 %1;	\
+				\
+		mtmsrd %0;	\
+		mfsprg2 %1;"
+	    : "=r"(scratch), "=r"(ppc64));
+
+	/*
+	 * Now copy restorebridge into all the handlers, if necessary,
+	 * and set up the trap tables.
+	 */
+
+	if (ppc64) {
+		/* Patch the two instances of rfi -> rfid */
+		bcopy(&rfid_patch,&rfi_patch1,4);
+		bcopy(&rfid_patch,&rfi_patch2,4);
+
+		/*
+		 * Copy a code snippet to restore 32-bit bridge mode
+		 * to the top of every non-generic trap handler
+		 */
+
+		trap_offset += (size_t)&restorebridgesize;
+		bcopy(&restorebridge, (void *)EXC_RST, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_DSI, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_ALI, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_PGM, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_MCHK, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_TRC, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_BPT, trap_offset); 
+
+		/*
+		 * Set the common trap entry point to the one that
+		 * knows to restore 32-bit operation on execution.
+		 */
+
+		generictrap = &trapcode64;
+	} else {
+		generictrap = &trapcode;
+	}
+
 #ifdef SMP
-	bcopy(&rstcode,  (void *)EXC_RST,  (size_t)&rstsize);
+	bcopy(&rstcode, (void *)(EXC_RST + trap_offset),  (size_t)&rstsize);
 #else
-	bcopy(&trapcode, (void *)EXC_RST,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_RST,  (size_t)&trapsize);
 #endif
-	bcopy(&trapcode, (void *)EXC_MCHK, (size_t)&trapsize);
-	bcopy(&dsitrap,  (void *)EXC_DSI,  (size_t)&dsisize);
-	bcopy(&trapcode, (void *)EXC_ISI,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_EXI,  (size_t)&trapsize);
-	bcopy(&alitrap,  (void *)EXC_ALI,  (size_t)&alisize);
-	bcopy(&trapcode, (void *)EXC_PGM,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_FPU,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_DECR, (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_SC,   (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_TRC,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_FPA,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_VEC,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_VECAST, (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_THRM, (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_BPT,  (size_t)&trapsize);
+
 #ifdef KDB
-	bcopy(&dblow,   (void *)EXC_MCHK, (size_t)&dbsize);
-	bcopy(&dblow,   (void *)EXC_PGM,  (size_t)&dbsize);
-	bcopy(&dblow,   (void *)EXC_TRC,  (size_t)&dbsize);
-	bcopy(&dblow,   (void *)EXC_BPT,  (size_t)&dbsize);
+	bcopy(&dblow,	(void *)(EXC_MCHK + trap_offset), (size_t)&dbsize);
+	bcopy(&dblow,   (void *)(EXC_PGM + trap_offset),  (size_t)&dbsize);
+	bcopy(&dblow,   (void *)(EXC_TRC + trap_offset),  (size_t)&dbsize);
+	bcopy(&dblow,   (void *)(EXC_BPT + trap_offset),  (size_t)&dbsize);
+#else
+	bcopy(generictrap, (void *)EXC_MCHK, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_PGM,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_TRC,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_BPT,  (size_t)&trapsize);
 #endif
+	bcopy(&dsitrap,  (void *)(EXC_DSI + trap_offset),  (size_t)&dsisize);
+	bcopy(&alitrap,  (void *)(EXC_ALI + trap_offset),  (size_t)&alisize);
+	bcopy(generictrap, (void *)EXC_ISI,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_EXI,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_FPU,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_DECR, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_SC,   (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_FPA,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_VEC,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_VECAST, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_THRM, (size_t)&trapsize);
 	__syncicache(EXC_RSVD, EXC_LAST - EXC_RSVD);
 
 	/*
-	 * Make sure translation has been enabled
+	 * Restore MSR
 	 */
-	mtmsr(mfmsr() | PSL_IR|PSL_DR|PSL_ME|PSL_RI);
+	mtmsr(msr);
 	isync();
 
 	/*
 	 * Initialise virtual memory.
 	 */
-	pmap_mmu_install(MMU_TYPE_OEA, 0);		/* XXX temporary */
+	if (ppc64)
+		pmap_mmu_install(MMU_TYPE_G5, 0);
+	else
+		pmap_mmu_install(MMU_TYPE_OEA, 0);
+
 	pmap_bootstrap(startkernel, endkernel);
+	mtmsr(mfmsr() | PSL_IR|PSL_DR|PSL_ME|PSL_RI);
+	isync();
 
 	/*
 	 * Initialize params/tunables that are derived from memsize
diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index 158b2ab..71947fa 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -323,6 +323,7 @@ void moea_zero_page_area(mmu_t, vm_page_t, int, int);
 void moea_zero_page_idle(mmu_t, vm_page_t);
 void moea_activate(mmu_t, struct thread *);
 void moea_deactivate(mmu_t, struct thread *);
+void moea_cpu_bootstrap(mmu_t, int);
 void moea_bootstrap(mmu_t, vm_offset_t, vm_offset_t);
 void *moea_mapdev(mmu_t, vm_offset_t, vm_size_t);
 void moea_unmapdev(mmu_t, vm_offset_t, vm_size_t);
@@ -364,6 +365,7 @@ static mmu_method_t moea_methods[] = {
 
 	/* Internal interfaces */
 	MMUMETHOD(mmu_bootstrap,       	moea_bootstrap),
+	MMUMETHOD(mmu_cpu_bootstrap,   	moea_cpu_bootstrap),
 	MMUMETHOD(mmu_mapdev,		moea_mapdev),
 	MMUMETHOD(mmu_unmapdev,		moea_unmapdev),
 	MMUMETHOD(mmu_kextract,		moea_kextract),
@@ -617,7 +619,7 @@ om_cmp(const void *a, const void *b)
 }
 
 void
-pmap_cpu_bootstrap(int ap)
+moea_cpu_bootstrap(mmu_t mmup, int ap)
 {
 	u_int sdr;
 	int i;
@@ -709,6 +711,9 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	__asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl));
 	isync();
 
+	/* set global direct map flag */
+	hw_direct_map = 1;
+
 	mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
 	CTR0(KTR_PMAP, "moea_bootstrap: physical memory");
 
@@ -895,7 +900,7 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	kernel_pmap->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT;
 	kernel_pmap->pm_active = ~0;
 
-	pmap_cpu_bootstrap(0);
+	moea_cpu_bootstrap(mmup,0);
 
 	pmap_bootstrapped++;
 
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
new file mode 100644
index 0000000..64e8c75
--- /dev/null
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -0,0 +1,2443 @@
+/*-
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $
+ */
+/*-
+ * Copyright (C) 2001 Benno Rice.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Manages physical address maps.
+ *
+ * In addition to hardware address maps, this module is called upon to
+ * provide software-use-only maps which may or may not be stored in the
+ * same form as hardware maps.  These pseudo-maps are used to store
+ * intermediate results from copy operations to and from address spaces.
+ *
+ * Since the information managed by this module is also stored by the
+ * logical address mapping module, this module may throw away valid virtual
+ * to physical mappings at almost any time.  However, invalidations of
+ * mappings must be done as requested.
+ *
+ * In order to cope with hardware architectures which make virtual to
+ * physical map invalidates expensive, this module may delay invalidate
+ * reduced protection operations until such time as they are actually
+ * necessary.  This module is given full information as to which processors
+ * are currently using which maps, and to when physical maps must be made
+ * correct.
+ */
+
+#include "opt_kstack_pages.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/msgbuf.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/vmmeter.h>
+
+#include <sys/kdb.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_pager.h>
+#include <vm/uma.h>
+
+#include <machine/cpu.h>
+#include <machine/powerpc.h>
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/psl.h>
+#include <machine/bat.h>
+#include <machine/pte.h>
+#include <machine/sr.h>
+#include <machine/trap.h>
+#include <machine/mmuvar.h>
+
+#include "mmu_if.h"
+
+#define	MOEA_DEBUG
+
+#define TODO	panic("%s: not implemented", __func__);
+
+static __inline u_int32_t
+cntlzw(volatile u_int32_t a) {
+	u_int32_t b;
+	__asm ("cntlzw %0, %1" : "=r"(b) : "r"(a));
+	return b;
+}
+
+static __inline uint64_t
+va_to_vsid(pmap_t pm, vm_offset_t va)
+{
+	return ((pm->pm_sr[(uintptr_t)va >> ADDR_SR_SHFT]) & SR_VSID_MASK);
+}
+
+#define	TLBSYNC()	__asm __volatile("tlbsync; ptesync");
+#define	SYNC()		__asm __volatile("sync");
+#define	EIEIO()		__asm __volatile("eieio");
+
+/*
+ * The tlbie instruction must be executed in 64-bit mode
+ * so we have to twiddle MSR[SF] around every invocation.
+ * Just to add to the fun, exceptions must be off as well
+ * so that we can't trap in 64-bit mode. What a pain.
+ */
+
+static __inline void
+TLBIE(pmap_t pmap, vm_offset_t va) {
+	register_t msr;
+	register_t scratch;
+
+	uint64_t vpn;
+	register_t vpn_hi, vpn_lo;
+
+#if 1
+	/*
+	 * CPU documentation says that tlbie takes the VPN, not the
+	 * VA. I think the code below does this correctly. We will see.
+	 */
+
+	vpn = (uint64_t)(va & ADDR_PIDX);
+	if (pmap != NULL)
+		vpn |= (va_to_vsid(pmap,va) << 28);
+#else
+	vpn = va;
+#endif
+
+	vpn_hi = (uint32_t)(vpn >> 32);
+	vpn_lo = (uint32_t)vpn;
+
+	__asm __volatile("\
+	    mfmsr %0; \
+	    clrldi %1,%0,49; \
+	    insrdi %1,1,1,0; \
+	    mtmsrd %1; \
+	    ptesync; \
+	    \
+	    sld %1,%2,%4; \
+	    or %1,%1,%3; \
+	    tlbie %1; \
+	    \
+	    mtmsrd %0; \
+	    eieio; \
+	    tlbsync; \
+	    ptesync;" 
+	: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32));
+}
+
+#define DISABLE_TRANS(msr)	msr = mfmsr(); mtmsr(msr & ~PSL_DR); isync()
+#define ENABLE_TRANS(msr)	mtmsr(msr); isync()
+
+#define	VSID_MAKE(sr, hash)	((sr) | (((hash) & 0xfffff) << 4))
+#define	VSID_TO_SR(vsid)	((vsid) & 0xf)
+#define	VSID_TO_HASH(vsid)	(((vsid) >> 4) & 0xfffff)
+
+#define	PVO_PTEGIDX_MASK	0x007		/* which PTEG slot */
+#define	PVO_PTEGIDX_VALID	0x008		/* slot is valid */
+#define	PVO_WIRED		0x010		/* PVO entry is wired */
+#define	PVO_MANAGED		0x020		/* PVO entry is managed */
+#define	PVO_BOOTSTRAP		0x080		/* PVO entry allocated during
+						   bootstrap */
+#define PVO_FAKE		0x100		/* fictitious phys page */
+#define	PVO_VADDR(pvo)		((pvo)->pvo_vaddr & ~ADDR_POFF)
+#define PVO_ISFAKE(pvo)		((pvo)->pvo_vaddr & PVO_FAKE)
+#define	PVO_PTEGIDX_GET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
+#define	PVO_PTEGIDX_ISSET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
+#define	PVO_PTEGIDX_CLR(pvo)	\
+	((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
+#define	PVO_PTEGIDX_SET(pvo, i)	\
+	((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
+
+#define	MOEA_PVO_CHECK(pvo)
+
+#define LOCK_TABLE() mtx_lock(&moea64_table_mutex)
+#define UNLOCK_TABLE() mtx_unlock(&moea64_table_mutex);
+#define ASSERT_TABLE_LOCK() mtx_assert(&moea64_table_mutex, MA_OWNED)
+
+struct ofw_map {
+	vm_offset_t	om_va;
+	vm_size_t	om_len;
+	vm_offset_t	om_pa_hi;
+	vm_offset_t	om_pa_lo;
+	u_int		om_mode;
+};
+
+/*
+ * Map of physical memory regions.
+ */
+static struct	mem_region *regions;
+static struct	mem_region *pregions;
+extern u_int	phys_avail_count;
+extern int	regions_sz, pregions_sz;
+extern int	ofw_real_mode;
+static struct	ofw_map translations[64];
+
+extern struct pmap ofw_pmap;
+
+extern void bs_remap_earlyboot(void);
+
+
+/*
+ * Lock for the pteg and pvo tables.
+ */
+struct mtx	moea64_table_mutex;
+
+/*
+ * PTEG data.
+ */
+static struct	lpteg *moea64_pteg_table;
+u_int		moea64_pteg_count;
+u_int		moea64_pteg_mask;
+
+/*
+ * PVO data.
+ */
+struct	pvo_head *moea64_pvo_table;		/* pvo entries by pteg index */
+/* lists of unmanaged pages */
+struct	pvo_head moea64_pvo_kunmanaged =
+    LIST_HEAD_INITIALIZER(moea64_pvo_kunmanaged);
+struct	pvo_head moea64_pvo_unmanaged =
+    LIST_HEAD_INITIALIZER(moea64_pvo_unmanaged);
+
+uma_zone_t	moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */
+uma_zone_t	moea64_mpvo_zone; /* zone for pvo entries for managed pages */
+
+vm_offset_t	pvo_allocator_start;
+vm_offset_t	pvo_allocator_end;
+
+#define	BPVO_POOL_SIZE	327680
+static struct	pvo_entry *moea64_bpvo_pool;
+static int	moea64_bpvo_pool_index = 0;
+
+#define	VSID_NBPW	(sizeof(u_int32_t) * 8)
+static u_int	moea64_vsid_bitmap[NPMAPS / VSID_NBPW];
+
+static boolean_t moea64_initialized = FALSE;
+
+/*
+ * Statistics.
+ */
+u_int	moea64_pte_valid = 0;
+u_int	moea64_pte_overflow = 0;
+u_int	moea64_pvo_entries = 0;
+u_int	moea64_pvo_enter_calls = 0;
+u_int	moea64_pvo_remove_calls = 0;
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_valid, CTLFLAG_RD, 
+    &moea64_pte_valid, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_overflow, CTLFLAG_RD,
+    &moea64_pte_overflow, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_entries, CTLFLAG_RD, 
+    &moea64_pvo_entries, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_enter_calls, CTLFLAG_RD,
+    &moea64_pvo_enter_calls, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD,
+    &moea64_pvo_remove_calls, 0, "");
+
+vm_offset_t	moea64_scratchpage_va[2];
+struct	pvo_entry *moea64_scratchpage_pvo[2];
+struct	lpte 	*moea64_scratchpage_pte[2];
+struct	mtx	moea64_scratchpage_mtx;
+
+/*
+ * Allocate physical memory for use in moea64_bootstrap.
+ */
+static vm_offset_t	moea64_bootstrap_alloc(vm_size_t, u_int);
+
+/*
+ * PTE calls.
+ */
+static int		moea64_pte_insert(u_int, struct lpte *);
+
+/*
+ * PVO calls.
+ */
+static int	moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *,
+		    vm_offset_t, vm_offset_t, uint64_t, int, int);
+static void	moea64_pvo_remove(struct pvo_entry *, int);
+static struct	pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t, int *);
+static struct	lpte *moea64_pvo_to_pte(const struct pvo_entry *, int);
+
+/*
+ * Utility routines.
+ */
+static void		moea64_bridge_bootstrap(mmu_t mmup, 
+			    vm_offset_t kernelstart, vm_offset_t kernelend);
+static void		moea64_bridge_cpu_bootstrap(mmu_t, int ap);
+static void		moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t,
+			    vm_prot_t, boolean_t);
+static boolean_t	moea64_query_bit(vm_page_t, u_int64_t);
+static u_int		moea64_clear_bit(vm_page_t, u_int64_t, u_int64_t *);
+static void		moea64_kremove(mmu_t, vm_offset_t);
+static void		moea64_syncicache(pmap_t pmap, vm_offset_t va, 
+			    vm_offset_t pa);
+static void		tlbia(void);
+
+/*
+ * Kernel MMU interface
+ */
+void moea64_change_wiring(mmu_t, pmap_t, vm_offset_t, boolean_t);
+void moea64_clear_modify(mmu_t, vm_page_t);
+void moea64_clear_reference(mmu_t, vm_page_t);
+void moea64_copy_page(mmu_t, vm_page_t, vm_page_t);
+void moea64_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t);
+void moea64_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
+    vm_prot_t);
+void moea64_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
+vm_paddr_t moea64_extract(mmu_t, pmap_t, vm_offset_t);
+vm_page_t moea64_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t);
+void moea64_init(mmu_t);
+boolean_t moea64_is_modified(mmu_t, vm_page_t);
+boolean_t moea64_ts_referenced(mmu_t, vm_page_t);
+vm_offset_t moea64_map(mmu_t, vm_offset_t *, vm_offset_t, vm_offset_t, int);
+boolean_t moea64_page_exists_quick(mmu_t, pmap_t, vm_page_t);
+int moea64_page_wired_mappings(mmu_t, vm_page_t);
+void moea64_pinit(mmu_t, pmap_t);
+void moea64_pinit0(mmu_t, pmap_t);
+void moea64_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
+void moea64_qenter(mmu_t, vm_offset_t, vm_page_t *, int);
+void moea64_qremove(mmu_t, vm_offset_t, int);
+void moea64_release(mmu_t, pmap_t);
+void moea64_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t);
+void moea64_remove_all(mmu_t, vm_page_t);
+void moea64_remove_write(mmu_t, vm_page_t);
+void moea64_zero_page(mmu_t, vm_page_t);
+void moea64_zero_page_area(mmu_t, vm_page_t, int, int);
+void moea64_zero_page_idle(mmu_t, vm_page_t);
+void moea64_activate(mmu_t, struct thread *);
+void moea64_deactivate(mmu_t, struct thread *);
+void *moea64_mapdev(mmu_t, vm_offset_t, vm_size_t);
+void moea64_unmapdev(mmu_t, vm_offset_t, vm_size_t);
+vm_offset_t moea64_kextract(mmu_t, vm_offset_t);
+void moea64_kenter(mmu_t, vm_offset_t, vm_offset_t);
+boolean_t moea64_dev_direct_mapped(mmu_t, vm_offset_t, vm_size_t);
+boolean_t moea64_page_executable(mmu_t, vm_page_t);
+
+static mmu_method_t moea64_bridge_methods[] = {
+	MMUMETHOD(mmu_change_wiring,	moea64_change_wiring),
+	MMUMETHOD(mmu_clear_modify,	moea64_clear_modify),
+	MMUMETHOD(mmu_clear_reference,	moea64_clear_reference),
+	MMUMETHOD(mmu_copy_page,	moea64_copy_page),
+	MMUMETHOD(mmu_enter,		moea64_enter),
+	MMUMETHOD(mmu_enter_object,	moea64_enter_object),
+	MMUMETHOD(mmu_enter_quick,	moea64_enter_quick),
+	MMUMETHOD(mmu_extract,		moea64_extract),
+	MMUMETHOD(mmu_extract_and_hold,	moea64_extract_and_hold),
+	MMUMETHOD(mmu_init,		moea64_init),
+	MMUMETHOD(mmu_is_modified,	moea64_is_modified),
+	MMUMETHOD(mmu_ts_referenced,	moea64_ts_referenced),
+	MMUMETHOD(mmu_map,     		moea64_map),
+	MMUMETHOD(mmu_page_exists_quick,moea64_page_exists_quick),
+	MMUMETHOD(mmu_page_wired_mappings,moea64_page_wired_mappings),
+	MMUMETHOD(mmu_pinit,		moea64_pinit),
+	MMUMETHOD(mmu_pinit0,		moea64_pinit0),
+	MMUMETHOD(mmu_protect,		moea64_protect),
+	MMUMETHOD(mmu_qenter,		moea64_qenter),
+	MMUMETHOD(mmu_qremove,		moea64_qremove),
+	MMUMETHOD(mmu_release,		moea64_release),
+	MMUMETHOD(mmu_remove,		moea64_remove),
+	MMUMETHOD(mmu_remove_all,      	moea64_remove_all),
+	MMUMETHOD(mmu_remove_write,	moea64_remove_write),
+	MMUMETHOD(mmu_zero_page,       	moea64_zero_page),
+	MMUMETHOD(mmu_zero_page_area,	moea64_zero_page_area),
+	MMUMETHOD(mmu_zero_page_idle,	moea64_zero_page_idle),
+	MMUMETHOD(mmu_activate,		moea64_activate),
+	MMUMETHOD(mmu_deactivate,      	moea64_deactivate),
+
+	/* Internal interfaces */
+	MMUMETHOD(mmu_bootstrap,       	moea64_bridge_bootstrap),
+	MMUMETHOD(mmu_cpu_bootstrap,   	moea64_bridge_cpu_bootstrap),
+	MMUMETHOD(mmu_mapdev,		moea64_mapdev),
+	MMUMETHOD(mmu_unmapdev,		moea64_unmapdev),
+	MMUMETHOD(mmu_kextract,		moea64_kextract),
+	MMUMETHOD(mmu_kenter,		moea64_kenter),
+	MMUMETHOD(mmu_dev_direct_mapped,moea64_dev_direct_mapped),
+	MMUMETHOD(mmu_page_executable,	moea64_page_executable),
+
+	{ 0, 0 }
+};
+
+static mmu_def_t oea64_bridge_mmu = {
+	MMU_TYPE_G5,
+	moea64_bridge_methods,
+	0
+};
+MMU_DEF(oea64_bridge_mmu);
+
+static __inline u_int
+va_to_pteg(uint64_t vsid, vm_offset_t addr)
+{
+	u_int hash;
+
+	hash = vsid ^ (((uint64_t)addr & ADDR_PIDX) >>
+	    ADDR_PIDX_SHFT);
+	return (hash & moea64_pteg_mask);
+}
+
+static __inline struct pvo_head *
+pa_to_pvoh(vm_offset_t pa, vm_page_t *pg_p)
+{
+	struct	vm_page *pg;
+
+	pg = PHYS_TO_VM_PAGE(pa);
+
+	if (pg_p != NULL)
+		*pg_p = pg;
+
+	if (pg == NULL)
+		return (&moea64_pvo_unmanaged);
+
+	return (&pg->md.mdpg_pvoh);
+}
+
+static __inline struct pvo_head *
+vm_page_to_pvoh(vm_page_t m)
+{
+
+	return (&m->md.mdpg_pvoh);
+}
+
+static __inline void
+moea64_attr_clear(vm_page_t m, u_int64_t ptebit)
+{
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	m->md.mdpg_attrs &= ~ptebit;
+}
+
+static __inline u_int64_t
+moea64_attr_fetch(vm_page_t m)
+{
+
+	return (m->md.mdpg_attrs);
+}
+
+static __inline void
+moea64_attr_save(vm_page_t m, u_int64_t ptebit)
+{
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	m->md.mdpg_attrs |= ptebit;
+}
+
+static __inline int
+moea64_pte_compare(const struct lpte *pt, const struct lpte *pvo_pt)
+{
+	if (pt->pte_hi == pvo_pt->pte_hi)
+		return (1);
+
+	return (0);
+}
+
+static __inline int
+moea64_pte_match(struct lpte *pt, uint64_t vsid, vm_offset_t va, int which)
+{
+	return (pt->pte_hi & ~LPTE_VALID) ==
+	    ((vsid << LPTE_VSID_SHIFT) |
+	    ((uint64_t)(va >> ADDR_API_SHFT64) & LPTE_API) | which);
+}
+
+static __inline void
+moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va, 
+    uint64_t pte_lo)
+{
+	ASSERT_TABLE_LOCK();
+
+	/*
+	 * Construct a PTE.  Default to IMB initially.  Valid bit only gets
+	 * set when the real pte is set in memory.
+	 *
+	 * Note: Don't set the valid bit for correct operation of tlb update.
+	 */
+	pt->pte_hi = (vsid << LPTE_VSID_SHIFT) |
+	    (((uint64_t)(va & ADDR_PIDX) >> ADDR_API_SHFT64) & LPTE_API);
+
+	pt->pte_lo = pte_lo;
+}
+
+static __inline void
+moea64_pte_synch(struct lpte *pt, struct lpte *pvo_pt)
+{
+
+	ASSERT_TABLE_LOCK();
+
+	pvo_pt->pte_lo |= pt->pte_lo & (LPTE_REF | LPTE_CHG);
+}
+
+static __inline void
+moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit)
+{
+	ASSERT_TABLE_LOCK();
+
+	/*
+	 * As shown in Section 7.6.3.2.3
+	 */
+	pt->pte_lo &= ~ptebit;
+	TLBIE(pmap,va);
+}
+
+static __inline void
+moea64_pte_set(struct lpte *pt, struct lpte *pvo_pt)
+{
+
+	ASSERT_TABLE_LOCK();
+	pvo_pt->pte_hi |= LPTE_VALID;
+
+	/*
+	 * Update the PTE as defined in section 7.6.3.1.
+	 * Note that the REF/CHG bits are from pvo_pt and thus should have
+	 * been saved so this routine can restore them (if desired).
+	 */
+	pt->pte_lo = pvo_pt->pte_lo;
+	EIEIO();
+	pt->pte_hi = pvo_pt->pte_hi;
+	SYNC();
+	moea64_pte_valid++;
+}
+
+static __inline void
+moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+{
+	ASSERT_TABLE_LOCK();
+	pvo_pt->pte_hi &= ~LPTE_VALID;
+
+	/*
+	 * Force the reg & chg bits back into the PTEs.
+	 */
+	SYNC();
+
+	/*
+	 * Invalidate the pte.
+	 */
+	pt->pte_hi &= ~LPTE_VALID;
+
+	TLBIE(pmap,va);
+
+	/*
+	 * Save the reg & chg bits.
+	 */
+	moea64_pte_synch(pt, pvo_pt);
+	moea64_pte_valid--;
+}
+
+static __inline void
+moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+{
+
+	/*
+	 * Invalidate the PTE
+	 */
+	moea64_pte_unset(pt, pvo_pt, pmap, va);
+	moea64_pte_set(pt, pvo_pt);
+}
+
+static __inline uint64_t
+moea64_calc_wimg(vm_offset_t pa)
+{
+	uint64_t pte_lo;
+	int i;
+
+	/*
+	 * Assume the page is cache inhibited and access is guarded unless
+	 * it's in our available memory array.
+	 */
+	pte_lo = LPTE_I | LPTE_G;
+	for (i = 0; i < pregions_sz; i++) {
+		if ((pa >= pregions[i].mr_start) &&
+		    (pa < (pregions[i].mr_start + pregions[i].mr_size))) {
+			pte_lo &= ~(LPTE_I | LPTE_G);
+			pte_lo |= LPTE_M;
+			break;
+		}
+	}
+
+	return pte_lo;
+}
+
+/*
+ * Quick sort callout for comparing memory regions.
+ */
+static int	mr_cmp(const void *a, const void *b);
+static int	om_cmp(const void *a, const void *b);
+
+static int
+mr_cmp(const void *a, const void *b)
+{
+	const struct	mem_region *regiona;
+	const struct	mem_region *regionb;
+
+	regiona = a;
+	regionb = b;
+	if (regiona->mr_start < regionb->mr_start)
+		return (-1);
+	else if (regiona->mr_start > regionb->mr_start)
+		return (1);
+	else
+		return (0);
+}
+
+static int
+om_cmp(const void *a, const void *b)
+{
+	const struct	ofw_map *mapa;
+	const struct	ofw_map *mapb;
+
+	mapa = a;
+	mapb = b;
+	if (mapa->om_pa_hi < mapb->om_pa_hi)
+		return (-1);
+	else if (mapa->om_pa_hi > mapb->om_pa_hi)
+		return (1);
+	else if (mapa->om_pa_lo < mapb->om_pa_lo)
+		return (-1);
+	else if (mapa->om_pa_lo > mapb->om_pa_lo)
+		return (1);
+	else
+		return (0);
+}
+
+static void
+moea64_bridge_cpu_bootstrap(mmu_t mmup, int ap)
+{
+	int i = 0;
+
+	/*
+	 * Initialize segment registers and MMU
+	 */
+
+	mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync();
+	for (i = 0; i < 16; i++) {
+		mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
+	}
+	__asm __volatile ("sync; mtsdr1 %0; isync"
+	    :: "r"((u_int)moea64_pteg_table 
+		     | (32 - cntlzw(moea64_pteg_mask >> 11))));
+	tlbia();
+}
+
+static void
+moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+{
+	ihandle_t	mmui;
+	phandle_t	chosen;
+	phandle_t	mmu;
+	int		sz;
+	int		i, j;
+	int		ofw_mappings;
+	vm_size_t	size, physsz, hwphyssz;
+	vm_offset_t	pa, va, off;
+	uint32_t	msr;
+
+	/* We don't have a direct map since there is no BAT */
+	hw_direct_map = 0;
+
+	/* Make sure battable is zero, since we have no BAT */
+	for (i = 0; i < 16; i++) {
+		battable[i].batu = 0;
+		battable[i].batl = 0;
+	}
+
+	/* Get physical memory regions from firmware */
+	mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
+	CTR0(KTR_PMAP, "moea64_bootstrap: physical memory");
+
+	qsort(pregions, pregions_sz, sizeof(*pregions), mr_cmp);
+	if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz)
+		panic("moea64_bootstrap: phys_avail too small");
+	qsort(regions, regions_sz, sizeof(*regions), mr_cmp);
+	phys_avail_count = 0;
+	physsz = 0;
+	hwphyssz = 0;
+	TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
+	for (i = 0, j = 0; i < regions_sz; i++, j += 2) {
+		CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start,
+		    regions[i].mr_start + regions[i].mr_size,
+		    regions[i].mr_size);
+		if (hwphyssz != 0 &&
+		    (physsz + regions[i].mr_size) >= hwphyssz) {
+			if (physsz < hwphyssz) {
+				phys_avail[j] = regions[i].mr_start;
+				phys_avail[j + 1] = regions[i].mr_start +
+				    hwphyssz - physsz;
+				physsz = hwphyssz;
+				phys_avail_count++;
+			}
+			break;
+		}
+		phys_avail[j] = regions[i].mr_start;
+		phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
+		phys_avail_count++;
+		physsz += regions[i].mr_size;
+	}
+	physmem = btoc(physsz);
+
+	/*
+	 * Allocate PTEG table.
+	 */
+#ifdef PTEGCOUNT
+	moea64_pteg_count = PTEGCOUNT;
+#else
+	moea64_pteg_count = 0x1000;
+
+	while (moea64_pteg_count < physmem)
+		moea64_pteg_count <<= 1;
+
+	moea64_pteg_count >>= 1;
+#endif /* PTEGCOUNT */
+
+	size = moea64_pteg_count * sizeof(struct lpteg);
+	CTR2(KTR_PMAP, "moea64_bootstrap: %d PTEGs, %d bytes", 
+	    moea64_pteg_count, size);
+
+	/*
+	 * We now need to allocate memory. This memory, to be allocated,
+	 * has to reside in a page table. The page table we are about to
+	 * allocate. We don't have BAT. So drop to data real mode for a minute
+	 * as a measure of last resort. We do this a couple times.
+	 */
+
+	moea64_pteg_table = (struct lpteg *)moea64_bootstrap_alloc(size, size);
+	DISABLE_TRANS(msr);
+	bzero((void *)moea64_pteg_table, moea64_pteg_count * sizeof(struct lpteg));
+	ENABLE_TRANS(msr);
+
+	moea64_pteg_mask = moea64_pteg_count - 1;
+
+	CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table);
+
+	/*
+	 * Allocate pv/overflow lists.
+	 */
+	size = sizeof(struct pvo_head) * moea64_pteg_count;
+
+	moea64_pvo_table = (struct pvo_head *)moea64_bootstrap_alloc(size,
+	    PAGE_SIZE);
+	CTR1(KTR_PMAP, "moea64_bootstrap: PVO table at %p", moea64_pvo_table);
+
+	DISABLE_TRANS(msr);
+	for (i = 0; i < moea64_pteg_count; i++)
+		LIST_INIT(&moea64_pvo_table[i]);
+	ENABLE_TRANS(msr);
+
+	/*
+	 * Initialize the lock that synchronizes access to the pteg and pvo
+	 * tables.
+	 */
+	mtx_init(&moea64_table_mutex, "pmap table", NULL, MTX_DEF |
+	    MTX_RECURSE);
+
+	/*
+	 * Initialise the unmanaged pvo pool.
+	 */
+	moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc(
+		BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0);
+	moea64_bpvo_pool_index = 0;
+
+	/*
+	 * Make sure kernel vsid is allocated as well as VSID 0.
+	 */
+	moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW]
+		|= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
+	moea64_vsid_bitmap[0] |= 1;
+
+	/*
+	 * Initialize the kernel pmap (which is statically allocated).
+	 */
+	for (i = 0; i < 16; i++) 
+		kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i;
+
+	kernel_pmap->pmap_phys = kernel_pmap;
+	kernel_pmap->pm_active = ~0;
+
+	PMAP_LOCK_INIT(kernel_pmap);
+
+	/*
+	 * Now map in all the other buffers we allocated earlier
+	 */
+
+	DISABLE_TRANS(msr);
+	size = moea64_pteg_count * sizeof(struct lpteg);
+	off = (vm_offset_t)(moea64_pteg_table);
+	for (pa = off; pa < off + size; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	size = sizeof(struct pvo_head) * moea64_pteg_count;
+	off = (vm_offset_t)(moea64_pvo_table);
+	for (pa = off; pa < off + size; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
+	off = (vm_offset_t)(moea64_bpvo_pool);
+	for (pa = off; pa < off + size; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	ENABLE_TRANS(msr);
+
+	/*
+	 * Map certain important things, like ourselves and the exception
+	 * vectors
+	 */
+
+	DISABLE_TRANS(msr);
+	for (pa = kernelstart & ~PAGE_MASK; pa < kernelend; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	for (pa = EXC_RSVD; pa < EXC_LAST; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	ENABLE_TRANS(msr);
+
+	if (!ofw_real_mode) {
+	    /*
+	     * Set up the Open Firmware pmap and add its mappings.
+	     */
+
+	    moea64_pinit(mmup, &ofw_pmap);
+	    ofw_pmap.pm_sr[KERNEL_SR] = kernel_pmap->pm_sr[KERNEL_SR];
+	    ofw_pmap.pm_sr[KERNEL2_SR] = kernel_pmap->pm_sr[KERNEL2_SR];
+
+	    if ((chosen = OF_finddevice("/chosen")) == -1)
+		panic("moea64_bootstrap: can't find /chosen");
+	    OF_getprop(chosen, "mmu", &mmui, 4);
+	    if ((mmu = OF_instance_to_package(mmui)) == -1)
+		panic("moea64_bootstrap: can't get mmu package");
+	    if ((sz = OF_getproplen(mmu, "translations")) == -1)
+		panic("moea64_bootstrap: can't get ofw translation count");
+
+	    bzero(translations, sz);
+	    if (OF_getprop(mmu, "translations", translations, sz) == -1)
+		panic("moea64_bootstrap: can't get ofw translations");
+
+	    CTR0(KTR_PMAP, "moea64_bootstrap: translations");
+	    sz /= sizeof(*translations);
+	    qsort(translations, sz, sizeof (*translations), om_cmp);
+
+	    for (i = 0, ofw_mappings = 0; i < sz; i++) {
+		CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x",
+		    (uint32_t)(translations[i].om_pa_lo), translations[i].om_va,
+		    translations[i].om_len);
+
+		if (translations[i].om_pa_lo % PAGE_SIZE)
+			panic("OFW translation not page-aligned!");
+
+		if (translations[i].om_pa_hi)
+			panic("OFW translations above 32-bit boundary!");
+
+		/* Now enter the pages for this mapping */
+
+		/*
+		 * Lock the ofw pmap. pmap_kenter(), which we use for the
+		 * pages the kernel also needs, does its own locking.
+		 */
+		PMAP_LOCK(&ofw_pmap); 
+		DISABLE_TRANS(msr);
+		for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) {
+			struct vm_page m;
+
+			/* Map low memory mappings into the kernel pmap, too.
+			 * These are typically mappings made by the loader,
+			 * so we need them if we want to keep executing. */
+
+			if (translations[i].om_va + off < SEGMENT_LENGTH)
+				moea64_kenter(mmup, translations[i].om_va + off,
+				    translations[i].om_va + off);
+
+			m.phys_addr = translations[i].om_pa_lo + off;
+			moea64_enter_locked(&ofw_pmap,
+			    translations[i].om_va + off, &m, VM_PROT_ALL, 1);
+
+			ofw_mappings++;
+		}
+		ENABLE_TRANS(msr);
+		PMAP_UNLOCK(&ofw_pmap);
+	    }
+	}
+
+#ifdef SMP
+	TLBSYNC();
+#endif
+
+	/*
+	 * Calculate the last available physical address.
+	 */
+	for (i = 0; phys_avail[i + 2] != 0; i += 2)
+		;
+	Maxmem = powerpc_btop(phys_avail[i + 1]);
+
+	/*
+	 * Initialize MMU and remap early physical mappings
+	 */
+	moea64_bridge_cpu_bootstrap(mmup,0);
+	mtmsr(mfmsr() | PSL_DR | PSL_IR); isync();
+	pmap_bootstrapped++;
+	bs_remap_earlyboot();
+
+	/*
+	 * Set the start and end of kva.
+	 */
+	virtual_avail = VM_MIN_KERNEL_ADDRESS;
+	virtual_end = VM_MAX_KERNEL_ADDRESS;
+
+	/*
+	 * Allocate some stupid buffer regions.
+	 */
+
+	pvo_allocator_start = virtual_avail;
+	virtual_avail += SEGMENT_LENGTH/4;
+	pvo_allocator_end = virtual_avail;
+
+	/*
+	 * Allocate some things for page zeroing
+	 */
+
+	mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL, MTX_DEF);
+	for (i = 0; i < 2; i++) {
+		moea64_scratchpage_va[i] = virtual_avail;
+		virtual_avail += PAGE_SIZE;
+
+		moea64_kenter(mmup,moea64_scratchpage_va[i],kernelstart);
+
+		LOCK_TABLE();
+		moea64_scratchpage_pvo[i] = moea64_pvo_find_va(kernel_pmap,
+		    moea64_scratchpage_va[i],&j);
+		moea64_scratchpage_pte[i] = moea64_pvo_to_pte(
+		    moea64_scratchpage_pvo[i],j);
+		UNLOCK_TABLE();
+	}
+
+	/*
+	 * Allocate a kernel stack with a guard page for thread0 and map it
+	 * into the kernel page map.
+	 */
+	pa = moea64_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, PAGE_SIZE);
+	va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
+	virtual_avail = va + KSTACK_PAGES * PAGE_SIZE;
+	CTR2(KTR_PMAP, "moea_bootstrap: kstack0 at %#x (%#x)", pa, va);
+	thread0.td_kstack = va;
+	thread0.td_kstack_pages = KSTACK_PAGES;
+	for (i = 0; i < KSTACK_PAGES; i++) {
+		moea64_kenter(mmup, va, pa);;
+		pa += PAGE_SIZE;
+		va += PAGE_SIZE;
+	}
+
+	/*
+	 * Allocate virtual address space for the message buffer.
+	 */
+	pa = msgbuf_phys = moea64_bootstrap_alloc(MSGBUF_SIZE, PAGE_SIZE);
+	msgbufp = (struct msgbuf *)virtual_avail;
+	va = virtual_avail;
+	virtual_avail += round_page(MSGBUF_SIZE);
+	while (va < virtual_avail) {
+		moea64_kenter(mmup, va, pa);;
+		pa += PAGE_SIZE;
+		va += PAGE_SIZE;
+	}
+}
+
+/*
+ * Activate a user pmap.  The pmap must be activated before it's address
+ * space can be accessed in any way.
+ */
+void
+moea64_activate(mmu_t mmu, struct thread *td)
+{
+	pmap_t	pm, pmr;
+
+	/*
+	 * Load all the data we need up front to encourage the compiler to
+	 * not issue any loads while we have interrupts disabled below.
+	 */
+	pm = &td->td_proc->p_vmspace->vm_pmap;
+	pmr = pm->pmap_phys;
+
+	pm->pm_active |= PCPU_GET(cpumask);
+	PCPU_SET(curpmap, pmr);
+}
+
+void
+moea64_deactivate(mmu_t mmu, struct thread *td)
+{
+	pmap_t	pm;
+
+	pm = &td->td_proc->p_vmspace->vm_pmap;
+	pm->pm_active &= ~(PCPU_GET(cpumask));
+	PCPU_SET(curpmap, NULL);
+}
+
+void
+moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
+{
+	struct	pvo_entry *pvo;
+
+	PMAP_LOCK(pm);
+	pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+
+	if (pvo != NULL) {
+		if (wired) {
+			if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
+				pm->pm_stats.wired_count++;
+			pvo->pvo_vaddr |= PVO_WIRED;
+		} else {
+			if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
+				pm->pm_stats.wired_count--;
+			pvo->pvo_vaddr &= ~PVO_WIRED;
+		}
+	}
+	PMAP_UNLOCK(pm);
+}
+
+/*
+ * Zero a page of physical memory by temporarily mapping it into the tlb.
+ */
+void
+moea64_zero_page(mmu_t mmu, vm_page_t m)
+{
+	moea64_zero_page_area(mmu,m,0,PAGE_SIZE);
+}
+
+/*
+ * This goes through and sets the physical address of our
+ * special scratch PTE to the PA we want to zero or copy. Because
+ * of locking issues (this can get called in pvo_enter() by
+ * the UMA allocator), we can't use most other utility functions here
+ */
+
+static __inline
+void moea64_set_scratchpage_pa(int which, vm_offset_t pa) {
+	moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo &= 
+	    (~LPTE_WIMG & ~LPTE_RPGN);
+	moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo |= 
+	    moea64_calc_wimg(pa) | (uint64_t)pa;
+
+	moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID;
+	TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+	
+	moea64_scratchpage_pte[which]->pte_lo = 
+	    moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo;
+	EIEIO();
+
+	moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID;
+	TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+}
+
+void
+moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst)
+{
+	vm_offset_t	dst;
+	vm_offset_t	src;
+
+	dst = VM_PAGE_TO_PHYS(mdst);
+	src = VM_PAGE_TO_PHYS(msrc);
+
+	mtx_lock(&moea64_scratchpage_mtx);
+
+	moea64_set_scratchpage_pa(0,src);
+	moea64_set_scratchpage_pa(1,dst);
+
+	kcopy((void *)moea64_scratchpage_va[0], 
+	    (void *)moea64_scratchpage_va[1], PAGE_SIZE);
+
+	__syncicache((void *)moea64_scratchpage_va[1],PAGE_SIZE);
+
+	mtx_unlock(&moea64_scratchpage_mtx);
+}
+
+void
+moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
+{
+	vm_offset_t pa = VM_PAGE_TO_PHYS(m);
+
+	if (!moea64_initialized)
+		panic("moea64_zero_page: can't zero pa %#x", pa);
+	if (size + off > PAGE_SIZE)
+		panic("moea64_zero_page: size + off > PAGE_SIZE");
+
+	mtx_lock(&moea64_scratchpage_mtx);
+
+	moea64_set_scratchpage_pa(0,pa);
+	bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
+	__syncicache((void *)moea64_scratchpage_va[0],PAGE_SIZE);
+
+	mtx_unlock(&moea64_scratchpage_mtx);
+}
+
+void
+moea64_zero_page_idle(mmu_t mmu, vm_page_t m)
+{
+
+	moea64_zero_page(mmu, m);
+}
+
+/*
+ * Map the given physical page at the specified virtual address in the
+ * target pmap with the protection requested.  If specified the page
+ * will be wired down.
+ */
+void
+moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 
+    vm_prot_t prot, boolean_t wired)
+{
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	moea64_enter_locked(pmap, va, m, prot, wired);
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pmap);
+}
+
+/*
+ * Map the given physical page at the specified virtual address in the
+ * target pmap with the protection requested.  If specified the page
+ * will be wired down.
+ *
+ * The page queues and pmap must be locked.
+ */
+
+static void
+moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+    boolean_t wired)
+{
+	struct		pvo_head *pvo_head;
+	uma_zone_t	zone;
+	vm_page_t	pg;
+	uint64_t	pte_lo;
+	u_int		pvo_flags;
+	int		error;
+
+	if (!moea64_initialized) {
+		pvo_head = &moea64_pvo_kunmanaged;
+		pg = NULL;
+		zone = moea64_upvo_zone;
+		pvo_flags = 0;
+	} else {
+		pvo_head = vm_page_to_pvoh(m);
+		pg = m;
+		zone = moea64_mpvo_zone;
+		pvo_flags = PVO_MANAGED;
+	}
+
+	if (pmap_bootstrapped)
+		mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+	/* XXX change the pvo head for fake pages */
+	if ((m->flags & PG_FICTITIOUS) == PG_FICTITIOUS) {
+		pvo_flags &= ~PVO_MANAGED;
+		pvo_head = &moea64_pvo_kunmanaged;
+		zone = moea64_upvo_zone;
+	}
+
+	pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m));
+
+	if (prot & VM_PROT_WRITE) {
+		pte_lo |= LPTE_BW;
+		if (pmap_bootstrapped)
+			vm_page_flag_set(m, PG_WRITEABLE);
+	} else
+		pte_lo |= LPTE_BR;
+
+	if (prot & VM_PROT_EXECUTE)
+		pvo_flags |= VM_PROT_EXECUTE;
+
+	if (wired)
+		pvo_flags |= PVO_WIRED;
+
+	if ((m->flags & PG_FICTITIOUS) != 0)
+		pvo_flags |= PVO_FAKE;
+
+	error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),
+	    pte_lo, pvo_flags, 0);
+
+	if (pmap == kernel_pmap)
+		TLBIE(pmap, va);
+
+	/*
+	 * Flush the page from the instruction cache if this page is
+	 * mapped executable and cacheable.
+	 */
+	if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+		moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m));
+	}
+}
+
+static void
+moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa)
+{
+	/*
+	 * This is much trickier than on older systems because
+	 * we can't sync the icache on physical addresses directly
+	 * without a direct map. Instead we check a couple of cases
+	 * where the memory is already mapped in and, failing that,
+	 * use the same trick we use for page zeroing to create
+	 * a temporary mapping for this physical address.
+	 */
+
+	if (!pmap_bootstrapped) {
+		/*
+		 * If PMAP is not bootstrapped, we are likely to be
+		 * in real mode.
+		 */
+		__syncicache((void *)pa,PAGE_SIZE);
+	} else if (pmap == kernel_pmap) {
+		__syncicache((void *)va,PAGE_SIZE);
+	} else {
+		/* Use the scratch page to set up a temp mapping */
+
+		mtx_lock(&moea64_scratchpage_mtx);
+
+		moea64_set_scratchpage_pa(1,pa);
+		__syncicache((void *)moea64_scratchpage_va[1],PAGE_SIZE);
+
+		mtx_unlock(&moea64_scratchpage_mtx);
+	}
+}
+
+/*
+ * Maps a sequence of resident pages belonging to the same object.
+ * The sequence begins with the given page m_start.  This page is
+ * mapped at the given virtual address start.  Each subsequent page is
+ * mapped at a virtual address that is offset from start by the same
+ * amount as the page is offset from m_start within the object.  The
+ * last page in the sequence is the page with the largest offset from
+ * m_start that can be mapped at a virtual address less than the given
+ * virtual address end.  Not every virtual page between start and end
+ * is mapped; only those for which a resident page exists with the
+ * corresponding offset from m_start are mapped.
+ */
+void
+moea64_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end,
+    vm_page_t m_start, vm_prot_t prot)
+{
+	vm_page_t m;
+	vm_pindex_t diff, psize;
+
+	psize = atop(end - start);
+	m = m_start;
+	PMAP_LOCK(pm);
+	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
+		moea64_enter_locked(pm, start + ptoa(diff), m, prot &
+		    (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
+		m = TAILQ_NEXT(m, listq);
+	}
+	PMAP_UNLOCK(pm);
+}
+
+void
+moea64_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m,
+    vm_prot_t prot)
+{
+	PMAP_LOCK(pm);
+	moea64_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE),
+	    FALSE);
+	PMAP_UNLOCK(pm);
+
+}
+
+vm_paddr_t
+moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va)
+{
+	struct	pvo_entry *pvo;
+	vm_paddr_t pa;
+
+	PMAP_LOCK(pm);
+	pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+	if (pvo == NULL)
+		pa = 0;
+	else
+		pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+	PMAP_UNLOCK(pm);
+	return (pa);
+}
+
+/*
+ * Atomically extract and hold the physical page with the given
+ * pmap and virtual address pair if that mapping permits the given
+ * protection.
+ */
+vm_page_t
+moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot)
+{
+	struct	pvo_entry *pvo;
+	vm_page_t m;
+        
+	m = NULL;
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);
+	if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
+	    ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW ||
+	     (prot & VM_PROT_WRITE) == 0)) {
+		m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+		vm_page_hold(m);
+	}
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pmap);
+	return (m);
+}
+
+static void *
+moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 
+{
+	/*
+	 * This entire routine is a horrible hack to avoid bothering kmem
+	 * for new KVA addresses. Because this can get called from inside
+	 * kmem allocation routines, calling kmem for a new address here
+	 * can lead to multiply locking non-recursive mutexes.
+	 */
+	static vm_pindex_t color;
+        vm_offset_t va;
+
+        vm_page_t m;
+        int pflags, needed_lock;
+
+	*flags = UMA_SLAB_PRIV;
+	needed_lock = !PMAP_LOCKED(kernel_pmap);
+
+	if (needed_lock)
+		PMAP_LOCK(kernel_pmap);
+
+        if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
+                pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
+        else
+                pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED;
+        if (wait & M_ZERO)
+                pflags |= VM_ALLOC_ZERO;
+
+        for (;;) {
+                m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ);
+                if (m == NULL) {
+                        if (wait & M_NOWAIT)
+                                return (NULL);
+                        VM_WAIT;
+                } else
+                        break;
+        }
+
+	va = pvo_allocator_start;
+	pvo_allocator_start += PAGE_SIZE;
+
+	if (pvo_allocator_start >= pvo_allocator_end)
+		panic("Ran out of PVO allocator buffer space!");
+
+	/* Now call pvo_enter in recursive mode */
+	moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+	    &moea64_pvo_kunmanaged, va,  VM_PAGE_TO_PHYS(m), LPTE_M, 
+	    PVO_WIRED | PVO_BOOTSTRAP, 1);
+
+	TLBIE(kernel_pmap, va);
+	
+	if (needed_lock)
+		PMAP_UNLOCK(kernel_pmap);
+
+	if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
+                bzero((void *)va, PAGE_SIZE);
+
+	return (void *)va;
+}
+
+void
+moea64_init(mmu_t mmu)
+{
+
+	CTR0(KTR_PMAP, "moea64_init");
+
+	moea64_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
+	    UMA_ZONE_VM | UMA_ZONE_NOFREE);
+	moea64_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
+	    UMA_ZONE_VM | UMA_ZONE_NOFREE);
+
+	if (!hw_direct_map) {
+		uma_zone_set_allocf(moea64_upvo_zone,moea64_uma_page_alloc);
+		uma_zone_set_allocf(moea64_mpvo_zone,moea64_uma_page_alloc);
+	}
+
+	moea64_initialized = TRUE;
+}
+
+boolean_t
+moea64_is_modified(mmu_t mmu, vm_page_t m)
+{
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return (FALSE);
+
+	return (moea64_query_bit(m, LPTE_CHG));
+}
+
+void
+moea64_clear_reference(mmu_t mmu, vm_page_t m)
+{
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return;
+	moea64_clear_bit(m, LPTE_REF, NULL);
+}
+
+void
+moea64_clear_modify(mmu_t mmu, vm_page_t m)
+{
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return;
+	moea64_clear_bit(m, LPTE_CHG, NULL);
+}
+
+/*
+ * Clear the write and modified bits in each of the given page's mappings.
+ */
+void
+moea64_remove_write(mmu_t mmu, vm_page_t m)
+{
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+	pmap_t	pmap;
+	uint64_t lo;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 ||
+	    (m->flags & PG_WRITEABLE) == 0)
+		return;
+	lo = moea64_attr_fetch(m);
+	SYNC();
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		pmap = pvo->pvo_pmap;
+		PMAP_LOCK(pmap);
+		if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
+			LOCK_TABLE();
+			pt = moea64_pvo_to_pte(pvo, -1);
+			pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
+			pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
+			if (pt != NULL) {
+				moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+				lo |= pvo->pvo_pte.lpte.pte_lo;
+				pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
+				moea64_pte_change(pt, &pvo->pvo_pte.lpte,
+				    pvo->pvo_pmap, pvo->pvo_vaddr);
+			}
+			UNLOCK_TABLE();
+		}
+		PMAP_UNLOCK(pmap);
+	}
+	if ((lo & LPTE_CHG) != 0) {
+		moea64_attr_clear(m, LPTE_CHG);
+		vm_page_dirty(m);
+	}
+	vm_page_flag_clear(m, PG_WRITEABLE);
+}
+
+/*
+ *	moea64_ts_referenced:
+ *
+ *	Return a count of reference bits for a page, clearing those bits.
+ *	It is not necessary for every reference bit to be cleared, but it
+ *	is necessary that 0 only be returned when there are truly no
+ *	reference bits set.
+ *
+ *	XXX: The exact number of bits to check and clear is a matter that
+ *	should be tested and standardized at some point in the future for
+ *	optimal aging of shared pages.
+ */
+boolean_t
+moea64_ts_referenced(mmu_t mmu, vm_page_t m)
+{
+	int count;
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return (0);
+
+	count = moea64_clear_bit(m, LPTE_REF, NULL);
+
+	return (count);
+}
+
+/*
+ * Map a wired page into kernel virtual address space.
+ */
+void
+moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
+{
+	uint64_t	pte_lo;
+	int		error;	
+
+	if (!pmap_bootstrapped) {
+		if (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)
+			panic("Trying to enter an address in KVA -- %#x!\n",pa);
+	}
+
+	pte_lo = moea64_calc_wimg(pa);
+
+	PMAP_LOCK(kernel_pmap);
+	error = moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+	    &moea64_pvo_kunmanaged, va, pa, pte_lo, 
+	    PVO_WIRED | VM_PROT_EXECUTE, 0);
+
+	TLBIE(kernel_pmap, va);
+
+	if (error != 0 && error != ENOENT)
+		panic("moea64_kenter: failed to enter va %#x pa %#x: %d", va,
+		    pa, error);
+
+	/*
+	 * Flush the memory from the instruction cache.
+	 */
+	if ((pte_lo & (LPTE_I | LPTE_G)) == 0) {
+		__syncicache((void *)va, PAGE_SIZE);
+	}
+	PMAP_UNLOCK(kernel_pmap);
+}
+
+/*
+ * Extract the physical page address associated with the given kernel virtual
+ * address.
+ */
+vm_offset_t
+moea64_kextract(mmu_t mmu, vm_offset_t va)
+{
+	struct		pvo_entry *pvo;
+	vm_paddr_t pa;
+
+	PMAP_LOCK(kernel_pmap);
+	pvo = moea64_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL);
+	KASSERT(pvo != NULL, ("moea64_kextract: no addr found"));
+	pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+	PMAP_UNLOCK(kernel_pmap);
+	return (pa);
+}
+
+/*
+ * Remove a wired page from kernel virtual address space.
+ */
+void
+moea64_kremove(mmu_t mmu, vm_offset_t va)
+{
+	moea64_remove(mmu, kernel_pmap, va, va + PAGE_SIZE);
+}
+
+/*
+ * Map a range of physical addresses into kernel virtual address space.
+ *
+ * The value passed in *virt is a suggested virtual address for the mapping.
+ * Architectures which can support a direct-mapped physical to virtual region
+ * can return the appropriate address within that region, leaving '*virt'
+ * unchanged.  We cannot and therefore do not; *virt is updated with the
+ * first usable address after the mapped region.
+ */
+vm_offset_t
+moea64_map(mmu_t mmu, vm_offset_t *virt, vm_offset_t pa_start,
+    vm_offset_t pa_end, int prot)
+{
+	vm_offset_t	sva, va;
+
+	sva = *virt;
+	va = sva;
+	for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE)
+		moea64_kenter(mmu, va, pa_start);
+	*virt = va;
+
+	return (sva);
+}
+
+/*
+ * Returns true if the pmap's pv is one of the first
+ * 16 pvs linked to from this page.  This count may
+ * be changed upwards or downwards in the future; it
+ * is only necessary that true be returned for a small
+ * subset of pmaps for proper page aging.
+ */
+boolean_t
+moea64_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m)
+{
+        int loops;
+	struct pvo_entry *pvo;
+
+        if (!moea64_initialized || (m->flags & PG_FICTITIOUS))
+                return FALSE;
+
+	loops = 0;
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		if (pvo->pvo_pmap == pmap)
+			return (TRUE);
+		if (++loops >= 16)
+			break;
+	}
+
+	return (FALSE);
+}
+
+/*
+ * Return the number of managed mappings to the given physical page
+ * that are wired.
+ */
+int
+moea64_page_wired_mappings(mmu_t mmu, vm_page_t m)
+{
+	struct pvo_entry *pvo;
+	int count;
+
+	count = 0;
+	if (!moea64_initialized || (m->flags & PG_FICTITIOUS) != 0)
+		return (count);
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink)
+		if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
+			count++;
+	return (count);
+}
+
+static u_int	moea64_vsidcontext;
+
+void
+moea64_pinit(mmu_t mmu, pmap_t pmap)
+{
+	int	i, mask;
+	u_int	entropy;
+
+	PMAP_LOCK_INIT(pmap);
+
+	entropy = 0;
+	__asm __volatile("mftb %0" : "=r"(entropy));
+
+	if (pmap_bootstrapped)
+		pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, (vm_offset_t)pmap);
+	else
+		pmap->pmap_phys = pmap;
+
+	/*
+	 * Allocate some segment registers for this pmap.
+	 */
+	for (i = 0; i < NPMAPS; i += VSID_NBPW) {
+		u_int	hash, n;
+
+		/*
+		 * Create a new value by mutiplying by a prime and adding in
+		 * entropy from the timebase register.  This is to make the
+		 * VSID more random so that the PT hash function collides
+		 * less often.  (Note that the prime casues gcc to do shifts
+		 * instead of a multiply.)
+		 */
+		moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy;
+		hash = moea64_vsidcontext & (NPMAPS - 1);
+		if (hash == 0)		/* 0 is special, avoid it */
+			continue;
+		n = hash >> 5;
+		mask = 1 << (hash & (VSID_NBPW - 1));
+		hash = (moea64_vsidcontext & 0xfffff);
+		if (moea64_vsid_bitmap[n] & mask) {	/* collision? */
+			/* anything free in this bucket? */
+			if (moea64_vsid_bitmap[n] == 0xffffffff) {
+				entropy = (moea64_vsidcontext >> 20);
+				continue;
+			}
+			i = ffs(~moea64_vsid_bitmap[i]) - 1;
+			mask = 1 << i;
+			hash &= 0xfffff & ~(VSID_NBPW - 1);
+			hash |= i;
+		}
+		moea64_vsid_bitmap[n] |= mask;
+		for (i = 0; i < 16; i++) {
+			pmap->pm_sr[i] = VSID_MAKE(i, hash);
+		}
+		return;
+	}
+
+	panic("moea64_pinit: out of segments");
+}
+
+/*
+ * Initialize the pmap associated with process 0.
+ */
+void
+moea64_pinit0(mmu_t mmu, pmap_t pm)
+{
+	moea64_pinit(mmu, pm);
+	bzero(&pm->pm_stats, sizeof(pm->pm_stats));
+}
+
+/*
+ * Set the physical protection on the specified range of this map as requested.
+ */
+void
+moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
+    vm_prot_t prot)
+{
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+	int	pteidx;
+
+	CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva,
+	    eva, prot);
+
+
+	KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap,
+	    ("moea64_protect: non current pmap"));
+
+	if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
+		moea64_remove(mmu, pm, sva, eva);
+		return;
+	}
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pm);
+	for (; sva < eva; sva += PAGE_SIZE) {
+		pvo = moea64_pvo_find_va(pm, sva, &pteidx);
+		if (pvo == NULL)
+			continue;
+
+		/*
+		 * Grab the PTE pointer before we diddle with the cached PTE
+		 * copy.
+		 */
+		LOCK_TABLE();
+		pt = moea64_pvo_to_pte(pvo, pteidx);
+
+		/*
+		 * Change the protection of the page.
+		 */
+		pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
+		pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
+		pvo->pvo_pte.lpte.pte_lo &= ~LPTE_NOEXEC;
+		if ((prot & VM_PROT_EXECUTE) == 0) 
+			pvo->pvo_pte.lpte.pte_lo |= LPTE_NOEXEC;
+
+		/*
+		 * If the PVO is in the page table, update that pte as well.
+		 */
+		if (pt != NULL) {
+			moea64_pte_change(pt, &pvo->pvo_pte.lpte, 
+			    pvo->pvo_pmap, pvo->pvo_vaddr);
+			if ((pvo->pvo_pte.lpte.pte_lo & 
+			    (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+				moea64_syncicache(pm, sva, 
+				     pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+			}
+		}
+		UNLOCK_TABLE();
+	}
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pm);
+}
+
+/*
+ * Map a list of wired pages into kernel virtual address space.  This is
+ * intended for temporary mappings which do not need page modification or
+ * references recorded.  Existing mappings in the region are overwritten.
+ */
+void
+moea64_qenter(mmu_t mmu, vm_offset_t va, vm_page_t *m, int count)
+{
+	while (count-- > 0) {
+		moea64_kenter(mmu, va, VM_PAGE_TO_PHYS(*m));
+		va += PAGE_SIZE;
+		m++;
+	}
+}
+
+/*
+ * Remove page mappings from kernel virtual address space.  Intended for
+ * temporary mappings entered by moea64_qenter.
+ */
+void
+moea64_qremove(mmu_t mmu, vm_offset_t va, int count)
+{
+	while (count-- > 0) {
+		moea64_kremove(mmu, va);
+		va += PAGE_SIZE;
+	}
+}
+
+void
+moea64_release(mmu_t mmu, pmap_t pmap)
+{
+        int idx, mask;
+        
+	/*
+	 * Free segment register's VSID
+	 */
+        if (pmap->pm_sr[0] == 0)
+                panic("moea64_release");
+
+        idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1);
+        mask = 1 << (idx % VSID_NBPW);
+        idx /= VSID_NBPW;
+        moea64_vsid_bitmap[idx] &= ~mask;
+	PMAP_LOCK_DESTROY(pmap);
+}
+
+/*
+ * Remove the given range of addresses from the specified map.
+ */
+void
+moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
+{
+	struct	pvo_entry *pvo;
+	int	pteidx;
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pm);
+	for (; sva < eva; sva += PAGE_SIZE) {
+		pvo = moea64_pvo_find_va(pm, sva, &pteidx);
+		if (pvo != NULL) {
+			moea64_pvo_remove(pvo, pteidx);
+		}
+	}
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pm);
+}
+
+/*
+ * Remove physical page from all pmaps in which it resides. moea64_pvo_remove()
+ * will reflect changes in pte's back to the vm_page.
+ */
+void
+moea64_remove_all(mmu_t mmu, vm_page_t m)
+{
+	struct  pvo_head *pvo_head;
+	struct	pvo_entry *pvo, *next_pvo;
+	pmap_t	pmap;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+
+	pvo_head = vm_page_to_pvoh(m);
+	for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
+		next_pvo = LIST_NEXT(pvo, pvo_vlink);
+
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+		pmap = pvo->pvo_pmap;
+		PMAP_LOCK(pmap);
+		moea64_pvo_remove(pvo, -1);
+		PMAP_UNLOCK(pmap);
+	}
+	vm_page_flag_clear(m, PG_WRITEABLE);
+}
+
+/*
+ * Allocate a physical page of memory directly from the phys_avail map.
+ * Can only be called from moea64_bootstrap before avail start and end are
+ * calculated.
+ */
+static vm_offset_t
+moea64_bootstrap_alloc(vm_size_t size, u_int align)
+{
+	vm_offset_t	s, e;
+	int		i, j;
+
+	size = round_page(size);
+	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
+		if (align != 0)
+			s = (phys_avail[i] + align - 1) & ~(align - 1);
+		else
+			s = phys_avail[i];
+		e = s + size;
+
+		if (s < phys_avail[i] || e > phys_avail[i + 1])
+			continue;
+
+		if (s == phys_avail[i]) {
+			phys_avail[i] += size;
+		} else if (e == phys_avail[i + 1]) {
+			phys_avail[i + 1] -= size;
+		} else {
+			for (j = phys_avail_count * 2; j > i; j -= 2) {
+				phys_avail[j] = phys_avail[j - 2];
+				phys_avail[j + 1] = phys_avail[j - 1];
+			}
+
+			phys_avail[i + 3] = phys_avail[i + 1];
+			phys_avail[i + 1] = s;
+			phys_avail[i + 2] = e;
+			phys_avail_count++;
+		}
+
+		return (s);
+	}
+	panic("moea64_bootstrap_alloc: could not allocate memory");
+}
+
+static void
+tlbia(void)
+{
+	vm_offset_t i;
+
+	for (i = 0; i < 0xFF000; i += 0x00001000) 
+		TLBIE(NULL,i);
+}
+
+static int
+moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
+    vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags, int recurse)
+{
+	struct	 pvo_entry *pvo;
+	uint64_t vsid;
+	int	 first;
+	u_int	 ptegidx;
+	int	 i;
+	int      bootstrap;
+
+	/*
+	 * One nasty thing that can happen here is that the UMA calls to
+	 * allocate new PVOs need to map more memory, which calls pvo_enter(),
+	 * which calls UMA...
+	 *
+	 * We break the loop by detecting recursion and allocating out of
+	 * the bootstrap pool.
+	 */
+
+	moea64_pvo_enter_calls++;
+	first = 0;
+	bootstrap = (flags & PVO_BOOTSTRAP);
+
+	if (!moea64_initialized)
+		bootstrap = 1;
+
+	/*
+	 * Compute the PTE Group index.
+	 */
+	va &= ~ADDR_POFF;
+	vsid = va_to_vsid(pm, va);
+	ptegidx = va_to_pteg(vsid, va);
+
+	/*
+	 * Remove any existing mapping for this page.  Reuse the pvo entry if
+	 * there is a mapping.
+	 */
+	if (!recurse)
+		LOCK_TABLE();
+
+	LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
+		if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
+			if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa &&
+			    (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) ==
+			    (pte_lo & LPTE_PP)) {
+				if (!recurse)
+					UNLOCK_TABLE();
+				return (0);
+			}
+			moea64_pvo_remove(pvo, -1);
+			break;
+		}
+	}
+
+	/*
+	 * If we aren't overwriting a mapping, try to allocate.
+	 */
+	if (bootstrap) {
+		if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) {
+			panic("moea64_enter: bpvo pool exhausted, %d, %d, %d",
+			      moea64_bpvo_pool_index, BPVO_POOL_SIZE, 
+			      BPVO_POOL_SIZE * sizeof(struct pvo_entry));
+		}
+		pvo = &moea64_bpvo_pool[moea64_bpvo_pool_index];
+		moea64_bpvo_pool_index++;
+		bootstrap = 1;
+	} else {
+		pvo = uma_zalloc(zone, M_NOWAIT);
+	}
+
+	if (pvo == NULL) {
+		if (!recurse)
+			UNLOCK_TABLE();
+		return (ENOMEM);
+	}
+
+	moea64_pvo_entries++;
+	pvo->pvo_vaddr = va;
+	pvo->pvo_pmap = pm;
+	LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink);
+	pvo->pvo_vaddr &= ~ADDR_POFF;
+
+	if (!(flags & VM_PROT_EXECUTE))
+		pte_lo |= LPTE_NOEXEC;
+	if (flags & PVO_WIRED)
+		pvo->pvo_vaddr |= PVO_WIRED;
+	if (pvo_head != &moea64_pvo_kunmanaged)
+		pvo->pvo_vaddr |= PVO_MANAGED;
+	if (bootstrap)
+		pvo->pvo_vaddr |= PVO_BOOTSTRAP;
+	if (flags & PVO_FAKE)
+		pvo->pvo_vaddr |= PVO_FAKE;
+
+	moea64_pte_create(&pvo->pvo_pte.lpte, vsid, va, 
+	    (uint64_t)(pa) | pte_lo);
+
+	/*
+	 * Remember if the list was empty and therefore will be the first
+	 * item.
+	 */
+	if (LIST_FIRST(pvo_head) == NULL)
+		first = 1;
+	LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
+
+	if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+		pm->pm_stats.wired_count++;
+	pm->pm_stats.resident_count++;
+
+	/*
+	 * We hope this succeeds but it isn't required.
+	 */
+	i = moea64_pte_insert(ptegidx, &pvo->pvo_pte.lpte);
+	if (i >= 0) {
+		PVO_PTEGIDX_SET(pvo, i);
+	} else {
+		panic("moea64_pvo_enter: overflow");
+		moea64_pte_overflow++;
+	}
+
+	if (!recurse)
+		UNLOCK_TABLE();
+
+	return (first ? ENOENT : 0);
+}
+
+static void
+moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
+{
+	struct	lpte *pt;
+
+	/*
+	 * If there is an active pte entry, we need to deactivate it (and
+	 * save the ref & cfg bits).
+	 */
+	LOCK_TABLE();
+	pt = moea64_pvo_to_pte(pvo, pteidx);
+	if (pt != NULL) {
+		moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap,
+		    pvo->pvo_vaddr);
+		PVO_PTEGIDX_CLR(pvo);
+	} else {
+		moea64_pte_overflow--;
+	}
+	UNLOCK_TABLE();
+
+	/*
+	 * Update our statistics.
+	 */
+	pvo->pvo_pmap->pm_stats.resident_count--;
+	if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+		pvo->pvo_pmap->pm_stats.wired_count--;
+
+	/*
+	 * Save the REF/CHG bits into their cache if the page is managed.
+	 */
+	if ((pvo->pvo_vaddr & (PVO_MANAGED|PVO_FAKE)) == PVO_MANAGED) {
+		struct	vm_page *pg;
+
+		pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+		if (pg != NULL) {
+			moea64_attr_save(pg, pvo->pvo_pte.lpte.pte_lo &
+			    (LPTE_REF | LPTE_CHG));
+		}
+	}
+
+	/*
+	 * Remove this PVO from the PV list.
+	 */
+	LIST_REMOVE(pvo, pvo_vlink);
+
+	/*
+	 * Remove this from the overflow list and return it to the pool
+	 * if we aren't going to reuse it.
+	 */
+	LIST_REMOVE(pvo, pvo_olink);
+	if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP))
+		uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea64_mpvo_zone :
+		    moea64_upvo_zone, pvo);
+	moea64_pvo_entries--;
+	moea64_pvo_remove_calls++;
+}
+
+static __inline int
+moea64_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
+{
+	int	pteidx;
+
+	/*
+	 * We can find the actual pte entry without searching by grabbing
+	 * the PTEG index from 3 unused bits in pte_lo[11:9] and by
+	 * noticing the HID bit.
+	 */
+	pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
+	if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
+		pteidx ^= moea64_pteg_mask * 8;
+
+	return (pteidx);
+}
+
+static struct pvo_entry *
+moea64_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
+{
+	struct		pvo_entry *pvo;
+	int		ptegidx;
+	uint64_t	vsid;
+
+	va &= ~ADDR_POFF;
+	vsid = va_to_vsid(pm, va);
+	ptegidx = va_to_pteg(vsid, va);
+
+	LOCK_TABLE();
+	LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
+		if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
+			if (pteidx_p)
+				*pteidx_p = moea64_pvo_pte_index(pvo, ptegidx);
+			break;
+		}
+	}
+	UNLOCK_TABLE();
+
+	return (pvo);
+}
+
+static struct lpte *
+moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
+{
+	struct lpte *pt;
+
+	/*
+	 * If we haven't been supplied the ptegidx, calculate it.
+	 */
+	if (pteidx == -1) {
+		int		ptegidx;
+		uint64_t	vsid;
+
+		vsid = va_to_vsid(pvo->pvo_pmap, pvo->pvo_vaddr);
+		ptegidx = va_to_pteg(vsid, pvo->pvo_vaddr);
+		pteidx = moea64_pvo_pte_index(pvo, ptegidx);
+	}
+
+	pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
+
+	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && 
+	    !PVO_PTEGIDX_ISSET(pvo)) {
+		panic("moea64_pvo_to_pte: pvo %p has valid pte in pvo but no "
+		    "valid pte index", pvo);
+	}
+
+	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0 && 
+	    PVO_PTEGIDX_ISSET(pvo)) {
+		panic("moea64_pvo_to_pte: pvo %p has valid pte index in pvo "
+		    "pvo but no valid pte", pvo);
+	}
+
+	if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) == 
+	    LPTE_VALID) {
+		if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) {
+			panic("moea64_pvo_to_pte: pvo %p has valid pte in "
+			    "moea64_pteg_table %p but invalid in pvo", pvo, pt);
+		}
+
+		if (((pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo) & 
+		    ~(LPTE_CHG|LPTE_REF)) != 0) {
+			panic("moea64_pvo_to_pte: pvo %p pte does not match "
+			    "pte %p in moea64_pteg_table difference is %#x", 
+			    pvo, pt,
+			    (uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo));
+		}
+
+		ASSERT_TABLE_LOCK();
+		return (pt);
+	}
+
+	if (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) {
+		panic("moea64_pvo_to_pte: pvo %p has invalid pte %p in "
+		    "moea64_pteg_table but valid in pvo", pvo, pt);
+	}
+
+	return (NULL);
+}
+
+static int
+moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
+{
+	struct	lpte *pt;
+	int	i;
+
+	ASSERT_TABLE_LOCK();
+
+	/*
+	 * First try primary hash.
+	 */
+	for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
+		if ((pt->pte_hi & LPTE_VALID) == 0) {
+			pvo_pt->pte_hi &= ~LPTE_HID;
+			moea64_pte_set(pt, pvo_pt);
+			return (i);
+		}
+	}
+
+	/*
+	 * Now try secondary hash.
+	 */
+	ptegidx ^= moea64_pteg_mask;
+
+	for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
+		if ((pt->pte_hi & LPTE_VALID) == 0) {
+			pvo_pt->pte_hi |= LPTE_HID;
+			moea64_pte_set(pt, pvo_pt);
+			return (i);
+		}
+	}
+
+	panic("moea64_pte_insert: overflow");
+	return (-1);
+}
+
+static boolean_t
+moea64_query_bit(vm_page_t m, u_int64_t ptebit)
+{
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+
+#if 0
+	if (moea64_attr_fetch(m) & ptebit)
+		return (TRUE);
+#endif
+
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+
+		/*
+		 * See if we saved the bit off.  If so, cache it and return
+		 * success.
+		 */
+		if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
+			moea64_attr_save(m, ptebit);
+			MOEA_PVO_CHECK(pvo);	/* sanity check */
+			return (TRUE);
+		}
+	}
+
+	/*
+	 * No luck, now go through the hard part of looking at the PTEs
+	 * themselves.  Sync so that any pending REF/CHG bits are flushed to
+	 * the PTEs.
+	 */
+	SYNC();
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+
+		/*
+		 * See if this pvo has a valid PTE.  if so, fetch the
+		 * REF/CHG bits from the valid PTE.  If the appropriate
+		 * ptebit is set, cache it and return success.
+		 */
+		LOCK_TABLE();
+		pt = moea64_pvo_to_pte(pvo, -1);
+		if (pt != NULL) {
+			moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+			if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
+				UNLOCK_TABLE();
+
+				moea64_attr_save(m, ptebit);
+				MOEA_PVO_CHECK(pvo);	/* sanity check */
+				return (TRUE);
+			}
+		}
+		UNLOCK_TABLE();
+	}
+
+	return (FALSE);
+}
+
+static u_int
+moea64_clear_bit(vm_page_t m, u_int64_t ptebit, u_int64_t *origbit)
+{
+	u_int	count;
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+	uint64_t rv;
+
+	/*
+	 * Clear the cached value.
+	 */
+	rv = moea64_attr_fetch(m);
+	moea64_attr_clear(m, ptebit);
+
+	/*
+	 * Sync so that any pending REF/CHG bits are flushed to the PTEs (so
+	 * we can reset the right ones).  note that since the pvo entries and
+	 * list heads are accessed via BAT0 and are never placed in the page
+	 * table, we don't have to worry about further accesses setting the
+	 * REF/CHG bits.
+	 */
+	SYNC();
+
+	/*
+	 * For each pvo entry, clear the pvo's ptebit.  If this pvo has a
+	 * valid pte clear the ptebit from the valid pte.
+	 */
+	count = 0;
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+
+		LOCK_TABLE();
+		pt = moea64_pvo_to_pte(pvo, -1);
+		if (pt != NULL) {
+			moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+			if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
+				count++;
+				moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit);
+			}
+		}
+		UNLOCK_TABLE();
+		rv |= pvo->pvo_pte.lpte.pte_lo;
+		pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+	}
+
+	if (origbit != NULL) {
+		*origbit = rv;
+	}
+
+	return (count);
+}
+
+boolean_t
+moea64_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size)
+{
+	return (EFAULT);
+}
+
+boolean_t
+moea64_page_executable(mmu_t mmu, vm_page_t pg)
+{
+	return (!moea64_query_bit(pg, LPTE_NOEXEC));
+}
+
+/*
+ * Map a set of physical memory pages into the kernel virtual
+ * address space. Return a pointer to where it is mapped. This
+ * routine is intended to be used for mapping device memory,
+ * NOT real memory.
+ */
+void *
+moea64_mapdev(mmu_t mmu, vm_offset_t pa, vm_size_t size)
+{
+	vm_offset_t va, tmpva, ppa, offset;
+
+	ppa = trunc_page(pa);
+	offset = pa & PAGE_MASK;
+	size = roundup(offset + size, PAGE_SIZE);
+
+	va = kmem_alloc_nofault(kernel_map, size);
+
+	if (!va)
+		panic("moea64_mapdev: Couldn't alloc kernel virtual memory");
+
+	for (tmpva = va; size > 0;) {
+		moea64_kenter(mmu, tmpva, ppa);
+		size -= PAGE_SIZE;
+		tmpva += PAGE_SIZE;
+		ppa += PAGE_SIZE;
+	}
+
+	return ((void *)(va + offset));
+}
+
+void
+moea64_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t base, offset;
+
+	base = trunc_page(va);
+	offset = va & PAGE_MASK;
+	size = roundup(offset + size, PAGE_SIZE);
+
+	kmem_free(kernel_map, base, size);
+}
+
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index cfdb53d..e226d3b 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -35,7 +35,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/smp.h>
 
-#include <machine/bat.h>
 #include <machine/bus.h>
 #include <machine/cpu.h>
 #include <machine/hid.h>
@@ -250,8 +249,10 @@ cpudep_ap_bootstrap(void)
 	mtmsr(msr);
 	isync();
 
-	reg = l3_enable();
-	reg = l2_enable();
+	if (l3cr_config != 0)
+		reg = l3_enable();
+	if (l2cr_config != 0)
+		reg = l2_enable();
 	reg = l1d_enable();
 	reg = l1i_enable();
 
diff --git a/sys/powerpc/aim/ofw_machdep.c b/sys/powerpc/aim/ofw_machdep.c
index 0d5f03b..977f81a 100644
--- a/sys/powerpc/aim/ofw_machdep.c
+++ b/sys/powerpc/aim/ofw_machdep.c
@@ -62,6 +62,12 @@ __FBSDID("$FreeBSD$");
 static struct mem_region OFmem[OFMEM_REGIONS + 1], OFavail[OFMEM_REGIONS + 3];
 static struct mem_region OFfree[OFMEM_REGIONS + 3];
 
+struct mem_region64 {
+        vm_offset_t     mr_start_hi;
+        vm_offset_t     mr_start_lo;
+        vm_size_t       mr_size;
+};	
+
 extern register_t ofmsr[5];
 extern struct	pmap ofw_pmap;
 static int	(*ofwcall)(void *);
@@ -141,24 +147,86 @@ void
 mem_regions(struct mem_region **memp, int *memsz,
 		struct mem_region **availp, int *availsz)
 {
-	int phandle;
+	phandle_t phandle;
 	int asz, msz, fsz;
 	int i, j;
 	int still_merging;
+	cell_t address_cells;
+
+	asz = msz = 0;
+
+	/*
+	 * Get #address-cells from root node, defaulting to 1 if it cannot
+	 * be found.
+	 */
+	phandle = OF_finddevice("/");
+	if (OF_getprop(phandle, "#address-cells", &address_cells, 
+	    sizeof(address_cells)) < sizeof(address_cells))
+		address_cells = 1;
 	
 	/*
 	 * Get memory.
 	 */
 	if ((phandle = OF_finddevice("/memory")) == -1
-	    || (msz = OF_getprop(phandle, "reg",
-			  OFmem, sizeof OFmem[0] * OFMEM_REGIONS))
-	       <= 0
 	    || (asz = OF_getprop(phandle, "available",
-			  OFavail, sizeof OFavail[0] * OFMEM_REGIONS))
-	       <= 0)
-		panic("no memory?");
+		  OFavail, sizeof OFavail[0] * OFMEM_REGIONS)) <= 0)
+	{
+		if (ofw_real_mode) {
+			/* XXX MAMBO */
+			printf("Physical memory unknown -- guessing 128 MB\n");
+
+			/* Leave the first 0xA000000 bytes for the kernel */
+			OFavail[0].mr_start = 0xA00000;
+			OFavail[0].mr_size = 0x75FFFFF;
+			asz = sizeof(OFavail[0]);
+		} else {
+			panic("no memory?");
+		}
+	}
+
+	if (address_cells == 2) {
+	    struct mem_region64 OFmem64[OFMEM_REGIONS + 1];
+	    if ((phandle == -1) || (msz = OF_getprop(phandle, "reg",
+			  OFmem64, sizeof OFmem64[0] * OFMEM_REGIONS)) <= 0) {
+		if (ofw_real_mode) {
+			/* XXX MAMBO */
+			OFmem64[0].mr_start_hi = 0;
+			OFmem64[0].mr_start_lo = 0x0;
+			OFmem64[0].mr_size = 0x7FFFFFF;
+			msz = sizeof(OFmem64[0]);
+		} else {
+			panic("Physical memory map not found");
+		}
+	    }
+
+	    for (i = 0, j = 0; i < msz/sizeof(OFmem64[0]); i++) {
+		if (OFmem64[i].mr_start_hi == 0) {
+			OFmem[i].mr_start = OFmem64[i].mr_start_lo;
+			OFmem[i].mr_size = OFmem64[i].mr_size;
+
+			/*
+			 * Check for memory regions extending above 32-bit
+			 * memory space, and restrict them to stay there.
+			 */
+			if (((uint64_t)OFmem[i].mr_start +
+			    (uint64_t)OFmem[i].mr_size) >
+			    BUS_SPACE_MAXADDR_32BIT) {
+				OFmem[i].mr_size = BUS_SPACE_MAXADDR_32BIT -
+				    OFmem[i].mr_start;
+			}
+			j++;
+		}
+	    }
+	    msz = j*sizeof(OFmem[0]);
+	} else {
+	    if ((msz = OF_getprop(phandle, "reg",
+			  OFmem, sizeof OFmem[0] * OFMEM_REGIONS)) <= 0)
+		panic("Physical memory map not found");
+	}
+
 	*memp = OFmem;
 	*memsz = msz / sizeof(struct mem_region);
+	
 
 	/*
 	 * OFavail may have overlapping regions - collapse these
@@ -268,8 +336,10 @@ openfirmware(void *args)
 		/*
 		 * Clear battable[] translations
 		 */
-		__asm __volatile("mtdbatu 2, %0\n"
-				 "mtdbatu 3, %0" : : "r" (0));
+		if (!ppc64) {
+			__asm __volatile("mtdbatu 2, %0\n"
+					 "mtdbatu 3, %0" : : "r" (0));
+		}
 		isync();
 	}
 
@@ -469,3 +539,4 @@ mem_valid(vm_offset_t addr, int len)
 
 	return (EFAULT);
 }
+
diff --git a/sys/powerpc/aim/trap_subr.S b/sys/powerpc/aim/trap_subr.S
index 58e0663..d782fae 100644
--- a/sys/powerpc/aim/trap_subr.S
+++ b/sys/powerpc/aim/trap_subr.S
@@ -223,10 +223,54 @@
 	lwz	%r3,(savearea+CPUSAVE_SRR0)(%r2); /* restore srr0 */	\
 	mtsrr0	%r3;							\
 	lwz	%r3,(savearea+CPUSAVE_SRR1)(%r2); /* restore srr1 */	\
+									\
+	/* Make sure HV bit of MSR propagated to SRR1 */		\
+	mfmsr	%r2;							\
+	or	%r3,%r2,%r3;						\
+									\
 	mtsrr1	%r3;							\
 	mfsprg2	%r2;			/* restore r2 & r3 */		\
 	mfsprg3	%r3
 
+/*
+ * The next two routines are 64-bit glue code. The first is used to test if
+ * we are on a 64-bit system. By copying it to the illegal instruction
+ * handler, we can test for 64-bit mode by trying to execute a 64-bit
+ * instruction and seeing what happens. The second gets copied in front
+ * of all the other handlers to restore 32-bit bridge mode when traps
+ * are taken.
+ */
+
+/* 64-bit test code. Sets SPRG2 to 0 if an illegal instruction is executed */
+
+	.globl	CNAME(testppc64),CNAME(testppc64size)
+CNAME(testppc64):
+	mtsprg1 %r31
+	mfsrr0  %r31
+	addi	%r31, %r31, 4
+	mtsrr0  %r31
+
+	li	%r31, 0
+	mtsprg2 %r31
+	mfsprg1 %r31
+
+	rfi
+CNAME(testppc64size) = .-CNAME(testppc64)
+
+
+/* 64-bit bridge mode restore snippet. Gets copied in front of everything else
+ * on 64-bit systems. */
+
+	.globl	CNAME(restorebridge),CNAME(restorebridgesize)
+CNAME(restorebridge):
+	mtsprg1	%r31
+	mfmsr	%r31
+	clrldi	%r31,%r31,1
+	mtmsrd	%r31
+	mfsprg1	%r31
+	isync
+CNAME(restorebridgesize) = .-CNAME(restorebridge)
+
 #ifdef SMP
 /*
  * Processor reset exception handler. These are typically
@@ -270,6 +314,17 @@ CNAME(trapcode):
 CNAME(trapsize) = .-CNAME(trapcode)
 
 /*
+ * 64-bit version of trapcode. Identical, except it calls generictrap64.
+ */
+	.globl	CNAME(trapcode64)
+CNAME(trapcode64):
+	mtsprg1	%r1			/* save SP */
+	mflr	%r1			/* Save the old LR in r1 */
+	mtsprg2 %r1			/* And then in SPRG2 */
+	li	%r1, 0x20		/* How to get the vector from LR */
+	bla	generictrap64		/* LR & SPRG3 is exception # */
+
+/*
  * For ALI: has to save DSISR and DAR
  */
 	.globl	CNAME(alitrap),CNAME(alisize)
@@ -433,6 +488,14 @@ realtrap:
  * SPRG2 - Original LR
  */
 
+generictrap64:
+	mtsprg3	%r31
+	mfmsr	%r31
+	clrldi	%r31,%r31,1
+	mtmsrd	%r31
+	mfsprg3	%r31
+	isync
+
 generictrap:
 	/* Save R1 for computing the exception vector */
 	mtsprg3 %r1
@@ -504,8 +567,15 @@ CNAME(asttrapexit):
 	b	trapexit		/* test ast ret value ? */
 1:
 	FRAME_LEAVE(PC_TEMPSAVE)
+
+	.globl	CNAME(rfi_patch1)	/* replace rfi with rfid on ppc64 */
+CNAME(rfi_patch1):
 	rfi
 
+	.globl	CNAME(rfid_patch)
+CNAME(rfid_patch):
+	rfid
+
 #if defined(KDB)
 /*
  * Deliberate entry to dbtrap
@@ -566,6 +636,8 @@ dbtrap:
 	b	realtrap
 dbleave:
 	FRAME_LEAVE(PC_DBSAVE)
+	.globl	CNAME(rfi_patch2)	/* replace rfi with rfid on ppc64 */
+CNAME(rfi_patch2):
 	rfi
 
 /*
diff --git a/sys/powerpc/aim/uio_machdep.c b/sys/powerpc/aim/uio_machdep.c
deleted file mode 100644
index 070ad98..0000000
--- a/sys/powerpc/aim/uio_machdep.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*-
- * Copyright (c) 2004 Alan L. Cox <alc@cs.rice.edu>
- * Copyright (c) 1982, 1986, 1991, 1993
- *	The Regents of the University of California.  All rights reserved.
- * (c) UNIX System Laboratories, Inc.
- * All or some portions of this file are derived from material licensed
- * to the University of California by American Telephone and Telegraph
- * Co. or Unix System Laboratories, Inc. and are reproduced herein with
- * the permission of UNIX System Laboratories, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)kern_subr.c	8.3 (Berkeley) 1/21/94
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/proc.h>
-#include <sys/systm.h>
-#include <sys/uio.h>
-
-#include <vm/vm.h>
-#include <vm/vm_page.h>
-
-#include <machine/md_var.h>
-#include <machine/vmparam.h>
-
-/*
- * Implement uiomove(9) from physical memory using the direct map to
- * avoid the creation and destruction of ephemeral mappings.
- */
-int
-uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
-{
-	struct thread *td = curthread;
-	struct iovec *iov;
-	void *cp;
-	vm_offset_t page_offset;
-	size_t cnt;
-	int error = 0;
-	int save = 0;
-
-	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
-	    ("uiomove_fromphys: mode"));
-	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
-	    ("uiomove_fromphys proc"));
-	save = td->td_pflags & TDP_DEADLKTREAT;
-	td->td_pflags |= TDP_DEADLKTREAT;
-	while (n > 0 && uio->uio_resid) {
-		iov = uio->uio_iov;
-		cnt = iov->iov_len;
-		if (cnt == 0) {
-			uio->uio_iov++;
-			uio->uio_iovcnt--;
-			continue;
-		}
-		if (cnt > n)
-			cnt = n;
-		page_offset = offset & PAGE_MASK;
-		cnt = min(cnt, PAGE_SIZE - page_offset);
-		cp = (char *)VM_PAGE_TO_PHYS(ma[offset >> PAGE_SHIFT]) +
-		    page_offset;
-		switch (uio->uio_segflg) {
-		case UIO_USERSPACE:
-			if (ticks - PCPU_GET(switchticks) >= hogticks)
-				uio_yield();
-			if (uio->uio_rw == UIO_READ)
-				error = copyout(cp, iov->iov_base, cnt);
-			else 
-				error = copyin(iov->iov_base, cp, cnt);
-			if (error)
-				goto out;
-			if (uio->uio_rw == UIO_WRITE &&
-			    pmap_page_executable(ma[offset >> PAGE_SHIFT]))
-				__syncicache(cp, cnt);
-			break;
-		case UIO_SYSSPACE:
-			if (uio->uio_rw == UIO_READ)
-				bcopy(cp, iov->iov_base, cnt);
-			else
-				bcopy(iov->iov_base, cp, cnt);
-			break;
-		case UIO_NOCOPY:
-			break;
-		}
-		iov->iov_base = (char *)iov->iov_base + cnt;
-		iov->iov_len -= cnt;
-		uio->uio_resid -= cnt;
-		uio->uio_offset += cnt;
-		offset += cnt;
-		n -= cnt;
-	}
-out:
-	if (save == 0)
-		td->td_pflags &= ~TDP_DEADLKTREAT;
-	return (error);
-}
diff --git a/sys/powerpc/aim/uma_machdep.c b/sys/powerpc/aim/uma_machdep.c
index 89d092a..dc03a26 100644
--- a/sys/powerpc/aim/uma_machdep.c
+++ b/sys/powerpc/aim/uma_machdep.c
@@ -35,9 +35,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <vm/vm.h>
 #include <vm/vm_page.h>
+#include <vm/vm_kern.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_extern.h>
+#include <vm/uma.h>
 #include <vm/uma.h>
 #include <vm/uma_int.h>
+#include <machine/md_var.h>
 #include <machine/vmparam.h>
 
 static int hw_uma_mdpages;
@@ -51,6 +55,13 @@ uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 	void *va;
 	vm_page_t m;
 	int pflags;
+	
+	if (!hw_direct_map) {
+		*flags = UMA_SLAB_KMEM;
+		va = (void *)kmem_malloc(kmem_map, bytes, wait);
+	
+		return va;
+	}
 
 	*flags = UMA_SLAB_PRIV;
 	if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
@@ -83,6 +94,12 @@ uma_small_free(void *mem, int size, u_int8_t flags)
 {
 	vm_page_t m;
 
+	if (!hw_direct_map) {
+		kmem_free(kmem_map, (vm_offset_t)mem, size);
+
+		return;
+	}
+
 	m = PHYS_TO_VM_PAGE((u_int32_t)mem);
 	m->wire_count--;
 	vm_page_free(m);
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c
index 9b0faf3..e20ba7b 100644
--- a/sys/powerpc/aim/vm_machdep.c
+++ b/sys/powerpc/aim/vm_machdep.c
@@ -101,6 +101,37 @@
 #include <vm/vm_extern.h>
 
 /*
+ * On systems without a direct mapped region (e.g. PPC64),
+ * we use the same code as the Book E implementation. Since
+ * we need to have runtime detection of this, define some machinery
+ * for sf_bufs in this case, and ignore it on systems with direct maps.
+ */
+
+#ifndef NSFBUFS
+#define NSFBUFS         (512 + maxusers * 16)
+#endif
+
+static void sf_buf_init(void *arg);
+SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL);
+ 
+LIST_HEAD(sf_head, sf_buf);
+ 
+/* A hash table of active sendfile(2) buffers */
+static struct sf_head *sf_buf_active;
+static u_long sf_buf_hashmask;
+
+#define SF_BUF_HASH(m)  (((m) - vm_page_array) & sf_buf_hashmask)
+
+static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
+static u_int sf_buf_alloc_want;
+
+/*
+ * A lock used to synchronize access to the hash table and free list
+ */
+static struct mtx sf_buf_lock;
+
+
+/*
  * Finish a fork operation, with process p2 nearly set up.
  * Copy and update the pcb, set up the stack so that the child
  * ready to run and return to user mode.
@@ -202,24 +233,122 @@ cpu_reset()
 }
 
 /*
- * Allocate an sf_buf for the given vm_page.  On this machine, however, there
- * is no sf_buf object.  Instead, an opaque pointer to the given vm_page is
- * returned.
+ * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
  */
-struct sf_buf *
-sf_buf_alloc(struct vm_page *m, int pri)
+static void
+sf_buf_init(void *arg)
 {
+        struct sf_buf *sf_bufs;
+        vm_offset_t sf_base;
+        int i;
+
+	/* Don't bother on systems with a direct map */
 
-	return ((struct sf_buf *)m);
+	if (hw_direct_map)
+		return;
+
+        nsfbufs = NSFBUFS;
+        TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
+
+        sf_buf_active = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
+        TAILQ_INIT(&sf_buf_freelist);
+        sf_base = kmem_alloc_nofault(kernel_map, nsfbufs * PAGE_SIZE);
+        sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP, M_NOWAIT | M_ZERO);
+
+        for (i = 0; i < nsfbufs; i++) {
+                sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
+                TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
+        }
+        sf_buf_alloc_want = 0;
+        mtx_init(&sf_buf_lock, "sf_buf", NULL, MTX_DEF);
+}
+
+/*
+ * Get an sf_buf from the freelist. Will block if none are available.
+ */
+struct sf_buf *
+sf_buf_alloc(struct vm_page *m, int flags)
+{
+        struct sf_head *hash_list;
+        struct sf_buf *sf;
+        int error;
+
+	if (hw_direct_map) {
+		/* Shortcut the direct mapped case */
+
+		return ((struct sf_buf *)m);
+	}
+
+        hash_list = &sf_buf_active[SF_BUF_HASH(m)];
+        mtx_lock(&sf_buf_lock);
+        LIST_FOREACH(sf, hash_list, list_entry) {
+                if (sf->m == m) {
+                        sf->ref_count++;
+                        if (sf->ref_count == 1) {
+                                TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
+                                nsfbufsused++;
+                                nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
+                        }
+                        goto done;
+                }
+        }
+
+        while ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
+                if (flags & SFB_NOWAIT)
+                        goto done;
+
+                sf_buf_alloc_want++;
+                mbstat.sf_allocwait++;
+                error = msleep(&sf_buf_freelist, &sf_buf_lock,
+                    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
+                sf_buf_alloc_want--;
+
+                /*
+                 * If we got a signal, don't risk going back to sleep.
+                 */
+                if (error)
+                        goto done;
+        }
+
+        TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
+        if (sf->m != NULL)
+                LIST_REMOVE(sf, list_entry);
+
+        LIST_INSERT_HEAD(hash_list, sf, list_entry);
+        sf->ref_count = 1;
+        sf->m = m;
+        nsfbufsused++;
+        nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
+        pmap_qenter(sf->kva, &sf->m, 1);
+done:
+        mtx_unlock(&sf_buf_lock);
+        return (sf);
 }
 
 /*
- * Free the sf_buf.  In fact, do nothing because there are no resources
- * associated with the sf_buf.
+ * Detatch mapped page and release resources back to the system.
+ *
+ * Remove a reference from the given sf_buf, adding it to the free
+ * list when its reference count reaches zero. A freed sf_buf still,
+ * however, retains its virtual-to-physical mapping until it is
+ * recycled or reactivated by sf_buf_alloc(9).
  */
 void
 sf_buf_free(struct sf_buf *sf)
 {
+	if (hw_direct_map)
+		return;
+
+        mtx_lock(&sf_buf_lock);
+        sf->ref_count--;
+        if (sf->ref_count == 0) {
+                TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
+                nsfbufsused--;
+
+                if (sf_buf_alloc_want > 0)
+                        wakeup_one(&sf_buf_freelist);
+        }
+        mtx_unlock(&sf_buf_lock);
 }
 
 /*
diff --git a/sys/powerpc/booke/machdep.c b/sys/powerpc/booke/machdep.c
index e42aa8c..bd72603 100644
--- a/sys/powerpc/booke/machdep.c
+++ b/sys/powerpc/booke/machdep.c
@@ -176,6 +176,9 @@ int cacheline_size = 32;
 SYSCTL_INT(_machdep, CPU_CACHELINE, cacheline_size,
 	   CTLFLAG_RD, &cacheline_size, 0, "");
 
+int hw_direct_map = 0;
+int ppc64 = 0;
+
 static void cpu_e500_startup(void *);
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_e500_startup, NULL);
 
diff --git a/sys/powerpc/include/hid.h b/sys/powerpc/include/hid.h
index d9fd6fe..52e5e62 100644
--- a/sys/powerpc/include/hid.h
+++ b/sys/powerpc/include/hid.h
@@ -47,6 +47,7 @@
 #define HID0_SLEEP	0x00200000  /* Enable sleep mode */
 #define HID0_DPM	0x00100000  /* Enable Dynamic power management */
 #define HID0_RISEG	0x00080000  /* Read I-SEG */
+#define HID0_TG		0x00040000  /* Timebase Granularity (OEA64) */
 #define HID0_BHTCLR	0x00040000  /* Clear branch history table (7450) */
 #define HID0_EIEC	0x00040000  /* Enable internal error checking */
 #define HID0_XAEN	0x00020000  /* Enable eXtended Addressing (7450) */
diff --git a/sys/powerpc/include/md_var.h b/sys/powerpc/include/md_var.h
index c554f5f..709e4a9 100644
--- a/sys/powerpc/include/md_var.h
+++ b/sys/powerpc/include/md_var.h
@@ -46,6 +46,8 @@ extern	u_long	ns_per_tick;
 
 extern	int powerpc_pow_enabled;
 extern	int cacheline_size;
+extern  int ppc64;
+extern  int hw_direct_map;
 
 void	__syncicache(void *, int);
 
diff --git a/sys/powerpc/include/sf_buf.h b/sys/powerpc/include/sf_buf.h
index 162f5a6..7ddb981 100644
--- a/sys/powerpc/include/sf_buf.h
+++ b/sys/powerpc/include/sf_buf.h
@@ -32,33 +32,9 @@
 #include <vm/vm.h>
 #include <vm/vm_param.h>
 #include <vm/vm_page.h>
+#include <machine/md_var.h>
 #include <sys/queue.h>
 
-#if defined(AIM)
-/*
- * On this machine, the only purpose for which sf_buf is used is to implement
- * an opaque pointer required by the machine-independent parts of the kernel.
- * That pointer references the vm_page that is "mapped" by the sf_buf.  The
- * actual mapping is provided by the direct virtual-to-physical mapping.  
- */
-struct sf_buf;
-
-static __inline vm_offset_t
-sf_buf_kva(struct sf_buf *sf)
-{
-
-	return (VM_PAGE_TO_PHYS((vm_page_t)sf));
-}
-
-static __inline vm_page_t
-sf_buf_page(struct sf_buf *sf)
-{
-
-	return ((vm_page_t)sf);
-}
-
-#elif defined(E500)
-
 struct vm_page;
 
 struct sf_buf {
@@ -69,9 +45,22 @@ struct sf_buf {
 	int		ref_count;	/* usage of this mapping */
 };
 
+/*
+ * On 32-bit OEA, the only purpose for which sf_buf is used is to implement
+ * an opaque pointer required by the machine-independent parts of the kernel.
+ * That pointer references the vm_page that is "mapped" by the sf_buf.  The
+ * actual mapping is provided by the direct virtual-to-physical mapping.  
+ *
+ * On OEA64 and Book-E, we need to do something a little more complicated. Use
+ * the runtime-detected hw_direct_map to pick between the two cases. Our
+ * friends in vm_machdep.c will do the same to ensure nothing gets confused.
+ */
+
 static __inline vm_offset_t
 sf_buf_kva(struct sf_buf *sf)
 {
+	if (hw_direct_map)
+		return (VM_PAGE_TO_PHYS((vm_page_t)sf));
 
 	return (sf->kva);
 }
@@ -79,10 +68,10 @@ sf_buf_kva(struct sf_buf *sf)
 static __inline struct vm_page *
 sf_buf_page(struct sf_buf *sf)
 {
+	if (hw_direct_map)
+		return ((vm_page_t)sf);
 
 	return (sf->m);
 }
 
-#endif
-
 #endif /* !_MACHINE_SF_BUF_H_ */
diff --git a/sys/powerpc/include/spr.h b/sys/powerpc/include/spr.h
index 826c2fa..eb07208 100644
--- a/sys/powerpc/include/spr.h
+++ b/sys/powerpc/include/spr.h
@@ -43,6 +43,44 @@
 	( { register_t val;						\
 	  __asm __volatile("mfspr %0,%1" : "=r"(val) : "K"(reg));	\
 	  val; } )
+
+/* The following routines allow manipulation of the full 64-bit width 
+ * of SPRs on 64 bit CPUs in bridge mode */
+
+#define mtspr64(reg,valhi,vallo,scratch)				\
+	__asm __volatile("						\
+		mfmsr %0; 						\
+		insrdi %0,1,1,0; 					\
+		mtmsrd %0; 						\
+		isync; 							\
+									\
+		sld %1,%1,%4;						\
+		or %1,%1,%2;						\
+		mtspr %3,%1;						\
+		srd %1,%1,%4;						\
+									\
+		clrldi %0,%0,1; 					\
+		mtmsrd %0; 						\
+		isync;"							\
+	: "=r"(scratch), "=r"(valhi) : "r"(vallo), "K"(reg), "r"(32))
+
+#define mfspr64upper(reg,scratch)					\
+	( { register_t val;						\
+	    __asm __volatile("						\
+		mfmsr %0; 						\
+		insrdi %0,1,1,0; 					\
+		mtmsrd %0; 						\
+		isync; 							\
+									\
+		mfspr %1,%2;						\
+		srd %1,%1,%3;						\
+									\
+		clrldi %0,%0,1; 					\
+		mtmsrd %0; 						\
+		isync;" 						\
+	    : "=r"(scratch), "=r"(val) : "K"(reg), "r"(32)); 			\
+	    val; } )
+
 #endif /* _LOCORE */
 
 /*
@@ -112,7 +150,11 @@
 #define	  IBM401E2		  0x0025
 #define	  IBM401F2		  0x0026
 #define	  IBM401G2		  0x0027
+#define	  IBM970		  0x0039
+#define	  IBM970FX		  0x003c
 #define	  IBMPOWER3		  0x0041
+#define	  IBM970MP		  0x0044
+#define	  IBM970GX		  0x0045
 #define	  MPC860		  0x0050
 #define	  MPC8240		  0x0081
 #define	  IBM405GP		  0x4011
diff --git a/sys/powerpc/include/vmparam.h b/sys/powerpc/include/vmparam.h
index 489159d..5af62eb 100644
--- a/sys/powerpc/include/vmparam.h
+++ b/sys/powerpc/include/vmparam.h
@@ -106,6 +106,13 @@
  */
 #define UMA_MD_SMALL_ALLOC
 
+/*
+ * On 64-bit systems in bridge mode, we have no direct map, so we fake
+ * the small_alloc() calls. But we need the VM to be in a reasonable
+ * state first.
+ */
+#define UMA_MD_SMALL_ALLOC_NEEDS_VM
+
 #else
 
 /*
diff --git a/sys/powerpc/ofw/ofw_syscons.c b/sys/powerpc/ofw/ofw_syscons.c
index 1080dad..5ab5939 100644
--- a/sys/powerpc/ofw/ofw_syscons.c
+++ b/sys/powerpc/ofw/ofw_syscons.c
@@ -216,6 +216,7 @@ ofwfb_configure(int flags)
         phandle_t chosen;
         ihandle_t stdout;
 	phandle_t node;
+	bus_addr_t fb_phys;
 	int depth;
 	int disable;
 	int len;
@@ -270,10 +271,16 @@ ofwfb_configure(int flags)
 	OF_getprop(node, "linebytes", &sc->sc_stride, sizeof(sc->sc_stride));
 
 	/*
-	 * XXX the physical address of the frame buffer is assumed to be
-	 * BAT-mapped so it can be accessed directly
+	 * Grab the physical address of the framebuffer, and then map it
+	 * into our memory space. If the MMU is not yet up, it will be
+	 * remapped for us when relocation turns on.
+	 *
+	 * XXX We assume #address-cells is 1 at this point.
 	 */
-	OF_getprop(node, "address", &sc->sc_addr, sizeof(sc->sc_addr));
+	OF_getprop(node, "address", &fb_phys, sizeof(fb_phys));
+
+	bus_space_map(&bs_be_tag, fb_phys, sc->sc_height * sc->sc_stride,
+	    0, &sc->sc_addr);
 
 	/*
 	 * Get the PCI addresses of the adapter. The node may be the
@@ -283,8 +290,8 @@ ofwfb_configure(int flags)
 	len = OF_getprop(node, "assigned-addresses", sc->sc_pciaddrs,
 	          sizeof(sc->sc_pciaddrs));
 	if (len == -1) {
-		len = OF_getprop(OF_parent(node), "assigned-addresses", sc->sc_pciaddrs,
-		          sizeof(sc->sc_pciaddrs));
+		len = OF_getprop(OF_parent(node), "assigned-addresses",
+		    sc->sc_pciaddrs, sizeof(sc->sc_pciaddrs));
 	}
 
 	if (len != -1) {
@@ -941,13 +948,17 @@ ofwfb_scidentify(driver_t *driver, device_t parent)
 static int
 ofwfb_scprobe(device_t dev)
 {
-	/* This is a fake device, so make sure there is no OF node for it */
-	if (ofw_bus_get_node(dev) != -1)
-		return ENXIO;
-	
+	int error;
+
 	device_set_desc(dev, "System console");
-	return (sc_probe_unit(device_get_unit(dev), 
-	    device_get_flags(dev) | SC_AUTODETECT_KBD));
+
+	error = sc_probe_unit(device_get_unit(dev), 
+	    device_get_flags(dev) | SC_AUTODETECT_KBD);
+	if (error != 0)
+		return (error);
+
+	/* This is a fake device, so make sure we added it ourselves */
+	return (BUS_PROBE_NOWILDCARD);
 }
 
 static int
diff --git a/sys/powerpc/powermac/cpcht.c b/sys/powerpc/powermac/cpcht.c
new file mode 100644
index 0000000..b8ab22f
--- /dev/null
+++ b/sys/powerpc/powermac/cpcht.c
@@ -0,0 +1,625 @@
+/*-
+ * Copyright (C) 2008 Nathan Whitehorn
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_pci.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include <machine/bus.h>
+#include <machine/md_var.h>
+#include <machine/pio.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <powerpc/powermac/cpchtvar.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include "pcib_if.h"
+
+#include "opt_isa.h"
+
+#ifdef DEV_ISA
+#include <isa/isavar.h>
+#endif
+
+static MALLOC_DEFINE(M_CPCHT, "cpcht", "CPC HT device information");
+
+/*
+ * HT Driver methods.
+ */
+static int		cpcht_probe(device_t);
+static int		cpcht_attach(device_t);
+static ofw_bus_get_devinfo_t cpcht_get_devinfo;
+
+
+static device_method_t	cpcht_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cpcht_probe),
+	DEVMETHOD(device_attach,	cpcht_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	bus_generic_read_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	bus_generic_alloc_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
+	DEVMETHOD(bus_activate_resource,bus_generic_activate_resource),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_devinfo,  cpcht_get_devinfo),
+	DEVMETHOD(ofw_bus_get_compat,   ofw_bus_gen_get_compat),
+	DEVMETHOD(ofw_bus_get_model,    ofw_bus_gen_get_model),
+	DEVMETHOD(ofw_bus_get_name,     ofw_bus_gen_get_name),
+	DEVMETHOD(ofw_bus_get_node,     ofw_bus_gen_get_node),
+	DEVMETHOD(ofw_bus_get_type,     ofw_bus_gen_get_type),
+
+	{ 0, 0 }
+};
+
+static driver_t	cpcht_driver = {
+	"cpcht",
+	cpcht_methods,
+	0
+};
+
+static devclass_t	cpcht_devclass;
+
+DRIVER_MODULE(cpcht, nexus, cpcht_driver, cpcht_devclass, 0, 0);
+
+static int 
+cpcht_probe(device_t dev) 
+{
+	const char	*type, *compatible;
+
+	type = ofw_bus_get_type(dev);
+	compatible = ofw_bus_get_compat(dev);
+
+	if (type == NULL || compatible == NULL)
+		return (ENXIO);
+
+	if (strcmp(type, "ht") != 0)
+		return (ENXIO);
+
+	if (strcmp(compatible, "u3-ht") == 0) {
+		device_set_desc(dev, "IBM CPC925 HyperTransport Tunnel");
+		return (0);
+	} else if (strcmp(compatible,"u4-ht") == 0) {
+		device_set_desc(dev, "IBM CPC945 HyperTransport Tunnel");
+		return (0);
+	}
+
+	return (ENXIO);
+}
+
+static int 
+cpcht_attach(device_t dev) 
+{
+	phandle_t root, child;
+	device_t cdev;
+	struct ofw_bus_devinfo *dinfo;
+	u_int32_t reg[6];
+
+	root = ofw_bus_get_node(dev);
+
+	if (OF_getprop(root, "reg", reg, sizeof(reg)) < 8)
+		return (ENXIO);
+
+	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+		dinfo = malloc(sizeof(*dinfo), M_CPCHT, M_WAITOK | M_ZERO);
+
+                if (ofw_bus_gen_setup_devinfo(dinfo, child) != 0) {
+                        free(dinfo, M_CPCHT);
+                        continue;
+                }
+                cdev = device_add_child(dev, NULL, -1);
+                if (cdev == NULL) {
+                        device_printf(dev, "<%s>: device_add_child failed\n",
+                            dinfo->obd_name);
+                        ofw_bus_gen_destroy_devinfo(dinfo);
+                        free(dinfo, M_CPCHT);
+                        continue;
+                }
+		device_set_ivars(cdev, dinfo);
+	}
+
+	return (bus_generic_attach(dev));
+}
+
+static const struct ofw_bus_devinfo *
+cpcht_get_devinfo(device_t dev, device_t child) 
+{
+	return (device_get_ivars(child));	
+}
+
+#ifdef DEV_ISA
+
+/*
+ * CPC ISA Device interface.
+ */
+static int		cpcisa_probe(device_t);
+
+/*
+ * Driver methods.
+ */
+static device_method_t	cpcisa_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cpcisa_probe),
+	DEVMETHOD(device_attach,	isab_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	bus_generic_read_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	bus_generic_alloc_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
+	DEVMETHOD(bus_activate_resource,bus_generic_activate_resource),
+
+	{0,0}
+};
+
+static driver_t	cpcisa_driver = {
+	"isab",
+	cpcisa_methods,
+	0
+};
+
+DRIVER_MODULE(cpcisa, cpcht, cpcisa_driver, isab_devclass, 0, 0);
+
+static int
+cpcisa_probe(device_t dev)
+{
+	const char	*type;
+
+	type = ofw_bus_get_type(dev);
+
+	if (type == NULL)
+		return (ENXIO);
+
+	if (strcmp(type, "isa") != 0)
+		return (ENXIO);
+
+	device_set_desc(dev, "HyperTransport-ISA bridge");
+	
+	return (0);
+}
+
+#endif /* DEV_ISA */
+
+/*
+ * CPC PCI Device interface.
+ */
+static int		cpcpci_probe(device_t);
+static int		cpcpci_attach(device_t);
+
+/*
+ * Bus interface.
+ */
+static int		cpcpci_read_ivar(device_t, device_t, int,
+			    uintptr_t *);
+static struct		resource * cpcpci_alloc_resource(device_t bus,
+			    device_t child, int type, int *rid, u_long start,
+			    u_long end, u_long count, u_int flags);
+static int		cpcpci_activate_resource(device_t bus, device_t child,
+			    int type, int rid, struct resource *res);
+
+/*
+ * pcib interface.
+ */
+static int		cpcpci_maxslots(device_t);
+static u_int32_t	cpcpci_read_config(device_t, u_int, u_int, u_int,
+			    u_int, int);
+static void		cpcpci_write_config(device_t, u_int, u_int, u_int,
+			    u_int, u_int32_t, int);
+static int		cpcpci_route_interrupt(device_t, device_t, int);
+
+/*
+ * ofw_bus interface
+ */
+
+static phandle_t	cpcpci_get_node(device_t bus, device_t child);
+
+/*
+ * Driver methods.
+ */
+static device_method_t	cpcpci_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cpcpci_probe),
+	DEVMETHOD(device_attach,	cpcpci_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	cpcpci_read_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	cpcpci_alloc_resource),
+	DEVMETHOD(bus_activate_resource,	cpcpci_activate_resource),
+
+	/* pcib interface */
+	DEVMETHOD(pcib_maxslots,	cpcpci_maxslots),
+	DEVMETHOD(pcib_read_config,	cpcpci_read_config),
+	DEVMETHOD(pcib_write_config,	cpcpci_write_config),
+	DEVMETHOD(pcib_route_interrupt,	cpcpci_route_interrupt),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_node,     cpcpci_get_node),
+	{ 0, 0 }
+};
+
+static driver_t	cpcpci_driver = {
+	"pcib",
+	cpcpci_methods,
+	sizeof(struct cpcpci_softc)
+};
+
+static devclass_t	cpcpci_devclass;
+
+DRIVER_MODULE(cpcpci, cpcht, cpcpci_driver, cpcpci_devclass, 0, 0);
+
+static int
+cpcpci_probe(device_t dev)
+{
+	const char	*type;
+
+	type = ofw_bus_get_type(dev);
+
+	if (type == NULL)
+		return (ENXIO);
+
+	if (strcmp(type, "pci") != 0)
+		return (ENXIO);
+
+	device_set_desc(dev, "HyperTransport-PCI bridge");
+	
+	return (0);
+}
+
+static int
+cpcpci_attach(device_t dev)
+{
+	struct		cpcpci_softc *sc;
+	phandle_t	node;
+	u_int32_t	reg[2], busrange[2], config_base;
+	struct		cpcpci_range *rp, *io, *mem[2];
+	struct		cpcpci_range fakeio;
+	int		nmem, i;
+
+	node = ofw_bus_get_node(dev);
+	sc = device_get_softc(dev);
+
+	if (OF_getprop(OF_parent(node), "reg", reg, sizeof(reg)) < 8)
+		return (ENXIO);
+
+	if (OF_getprop(node, "bus-range", busrange, sizeof(busrange)) != 8)
+		return (ENXIO);
+
+	sc->sc_dev = dev;
+	sc->sc_node = node;
+	sc->sc_bus = busrange[0];
+	config_base = reg[1];
+	if (sc->sc_bus)
+		config_base += 0x01000000UL + (sc->sc_bus << 16);
+	sc->sc_data = (vm_offset_t)pmap_mapdev(config_base, PAGE_SIZE << 4);
+
+	bzero(sc->sc_range, sizeof(sc->sc_range));
+	sc->sc_nrange = OF_getprop(node, "ranges", sc->sc_range,
+	    sizeof(sc->sc_range));
+
+	if (sc->sc_nrange == -1) {
+		device_printf(dev, "could not get ranges\n");
+		return (ENXIO);
+	}
+
+	sc->sc_range[6].pci_hi = 0;
+	io = NULL;
+	nmem = 0;
+
+	for (rp = sc->sc_range; rp->pci_hi != 0; rp++) {
+		switch (rp->pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) {
+		case OFW_PCI_PHYS_HI_SPACE_CONFIG:
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_IO:
+			io = rp;
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_MEM32:
+			mem[nmem] = rp;
+			nmem++;
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_MEM64:
+			break;
+		}
+	}
+
+	if (io == NULL) {
+		/*
+		 * On at least some machines, the I/O port range is
+		 * not exported in the OF device tree. So hardcode it.
+		 */
+
+		fakeio.host_lo = 0;
+		fakeio.pci_lo = reg[1];
+		fakeio.size_lo = 0x00400000;
+		if (sc->sc_bus)
+			fakeio.pci_lo += 0x02000000UL + (sc->sc_bus << 14);
+		io = &fakeio;
+	}
+	sc->sc_io_rman.rm_type = RMAN_ARRAY;
+	sc->sc_io_rman.rm_descr = "CPC 9xx PCI I/O Ports";
+	sc->sc_iostart = io->host_lo;
+	if (rman_init(&sc->sc_io_rman) != 0 ||
+	    rman_manage_region(&sc->sc_io_rman, io->pci_lo,
+	    io->pci_lo + io->size_lo - 1) != 0) {
+		device_printf(dev, "failed to set up io range management\n");
+		return (ENXIO);
+	}
+
+	if (nmem == 0) {
+		device_printf(dev, "can't find mem ranges\n");
+		return (ENXIO);
+	}
+	sc->sc_mem_rman.rm_type = RMAN_ARRAY;
+	sc->sc_mem_rman.rm_descr = "CPC 9xx PCI Memory";
+	if (rman_init(&sc->sc_mem_rman) != 0) {
+		device_printf(dev,
+		    "failed to init mem range resources\n");
+		return (ENXIO);
+	}
+	for (i = 0; i < nmem; i++) {
+		if (rman_manage_region(&sc->sc_mem_rman, mem[i]->pci_lo,
+		    mem[i]->pci_lo + mem[i]->size_lo - 1) != 0) {
+			device_printf(dev,
+			    "failed to set up memory range management\n");
+			return (ENXIO);
+		}
+	}
+
+	ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(cell_t));
+
+	device_add_child(dev, "pci", device_get_unit(dev));
+
+	return (bus_generic_attach(dev));
+}
+
+static int
+cpcpci_maxslots(device_t dev)
+{
+
+	return (PCI_SLOTMAX);
+}
+
+static u_int32_t
+cpcpci_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
+    int width)
+{
+	struct		cpcpci_softc *sc;
+	vm_offset_t	caoff;
+
+	sc = device_get_softc(dev);
+	caoff = sc->sc_data + 
+		(((((slot & 0x1f) << 3) | (func & 0x07)) << 8) | reg);
+
+	switch (width) {
+	case 1:
+		return (in8rb(caoff));
+		break;
+	case 2:
+		return (in16rb(caoff));
+		break;
+	case 4:
+		return (in32rb(caoff));
+		break;
+	}
+
+	return (0xffffffff);
+}
+
+static void
+cpcpci_write_config(device_t dev, u_int bus, u_int slot, u_int func,
+    u_int reg, u_int32_t val, int width)
+{
+	struct		cpcpci_softc *sc;
+	vm_offset_t	caoff;
+
+	sc = device_get_softc(dev);
+	caoff = sc->sc_data + 
+		(((((slot & 0x1f) << 3) | (func & 0x07)) << 8) | reg);
+
+	switch (width) {
+	case 1:
+		out8rb(caoff, val);
+		break;
+	case 2:
+		out16rb(caoff, val);
+		break;
+	case 4:
+		out32rb(caoff, val);
+		break;
+	}
+}
+
+static int
+cpcpci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
+{
+	struct	cpcpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	switch (which) {
+	case PCIB_IVAR_DOMAIN:
+		*result = device_get_unit(dev);
+		return (0);
+	case PCIB_IVAR_BUS:
+		*result = sc->sc_bus;
+		return (0);
+	}
+
+	return (ENOENT);
+}
+
+static struct resource *
+cpcpci_alloc_resource(device_t bus, device_t child, int type, int *rid,
+    u_long start, u_long end, u_long count, u_int flags)
+{
+	struct			cpcpci_softc *sc;
+	struct			resource *rv;
+	struct			rman *rm;
+	int			needactivate;
+
+	needactivate = flags & RF_ACTIVE;
+	flags &= ~RF_ACTIVE;
+
+	sc = device_get_softc(bus);
+
+	switch (type) {
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_mem_rman;
+		break;
+
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_io_rman;
+		if (rm == NULL)
+			return (NULL);
+		break;
+
+	case SYS_RES_IRQ:
+		return (bus_alloc_resource(bus, type, rid, start, end, count,
+		    flags));
+
+	default:
+		device_printf(bus, "unknown resource request from %s\n",
+		    device_get_nameunit(child));
+		return (NULL);
+	}
+
+	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	if (rv == NULL) {
+		device_printf(bus, "failed to reserve resource for %s\n",
+		    device_get_nameunit(child));
+		return (NULL);
+	}
+
+	rman_set_rid(rv, *rid);
+
+	if (needactivate) {
+		if (bus_activate_resource(child, type, *rid, rv) != 0) {
+			device_printf(bus,
+			    "failed to activate resource for %s\n",
+			    device_get_nameunit(child));
+			rman_release_resource(rv);
+			return (NULL);
+		}
+	}
+
+	return (rv);
+}
+
+static int
+cpcpci_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *res)
+{
+	void	*p;
+	struct	cpcpci_softc *sc;
+
+	sc = device_get_softc(bus);
+
+	if (type == SYS_RES_IRQ)
+		return (bus_activate_resource(bus, type, rid, res));
+
+	if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) {
+		vm_offset_t start;
+
+		start = (vm_offset_t)rman_get_start(res);
+		/*
+		 * For i/o-ports, convert the start address to the
+		 * CPC PCI i/o window
+		 */
+		if (type == SYS_RES_IOPORT)
+			start += sc->sc_iostart;
+
+		if (bootverbose)
+			printf("cpcpci mapdev: start %x, len %ld\n", start,
+			    rman_get_size(res));
+
+		p = pmap_mapdev(start, (vm_size_t)rman_get_size(res));
+		if (p == NULL)
+			return (ENOMEM);
+		rman_set_virtual(res, p);
+		rman_set_bustag(res, &bs_le_tag);
+		rman_set_bushandle(res, (u_long)p);
+	}
+
+	return (rman_activate_resource(res));
+}
+
+static phandle_t
+cpcpci_get_node(device_t bus, device_t dev)
+{
+	struct cpcpci_softc *sc;
+
+	sc = device_get_softc(bus);
+	/* We only have one child, the PCI bus, which needs our own node. */
+	return (sc->sc_node);
+}
+
+static int
+cpcpci_route_interrupt(device_t bus, device_t dev, int pin)
+{
+	struct cpcpci_softc *sc;
+	struct ofw_pci_register reg;
+	uint32_t pintr, mintr;
+	uint8_t maskbuf[sizeof(reg) + sizeof(pintr)];
+
+	sc = device_get_softc(bus);
+	pintr = pin;
+	if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo, &reg,
+	    sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr), maskbuf))
+		return (mintr);
+
+	/* Maybe it's a real interrupt, not an intpin */
+	if (pin > 4)
+		return (pin);
+
+	device_printf(bus, "could not route pin %d for device %d.%d\n",
+	    pin, pci_get_slot(dev), pci_get_function(dev));
+	return (PCI_INVALID_IRQ);
+}
+
diff --git a/sys/powerpc/powermac/cpchtvar.h b/sys/powerpc/powermac/cpchtvar.h
new file mode 100644
index 0000000..270345f
--- /dev/null
+++ b/sys/powerpc/powermac/cpchtvar.h
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (C) 2008 Nathan Whitehorn
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_POWERPC_POWERMAC_CPCHTVAR_H_
+#define	_POWERPC_POWERMAC_CPCHTVAR_H_
+
+struct cpcpci_range {
+	u_int32_t	pci_hi;
+	u_int32_t	pci_mid;
+	u_int32_t	pci_lo;
+	u_int32_t	junk;
+	u_int32_t	host_hi;
+	u_int32_t	host_lo;
+	u_int32_t	size_hi;
+	u_int32_t	size_lo;
+};
+
+struct cpcpci_softc {
+	device_t		sc_dev;
+	phandle_t		sc_node;
+	vm_offset_t		sc_data;
+	int			sc_bus;
+	struct			cpcpci_range sc_range[6];
+	int			sc_nrange;
+	int			sc_iostart;
+	struct			rman sc_io_rman;
+	struct			rman sc_mem_rman;
+	bus_space_tag_t		sc_iot;
+	bus_space_tag_t		sc_memt;
+	bus_dma_tag_t		sc_dmat;
+	struct ofw_bus_iinfo	sc_pci_iinfo;
+};
+
+#endif  /* _POWERPC_POWERMAC_CPCHTVAR_H_ */
diff --git a/sys/powerpc/powerpc/bus_machdep.c b/sys/powerpc/powerpc/bus_machdep.c
index 42e5baa..742454b 100644
--- a/sys/powerpc/powerpc/bus_machdep.c
+++ b/sys/powerpc/powerpc/bus_machdep.c
@@ -42,12 +42,25 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/pio.h>
+#include <machine/md_var.h>
 
 #define TODO panic("%s: not implemented", __func__)
 
+#define	MAX_EARLYBOOT_MAPPINGS	6
+
+static struct {
+	bus_addr_t addr;
+	bus_size_t size;
+} earlyboot_mappings[MAX_EARLYBOOT_MAPPINGS];
+static int earlyboot_map_idx = 0;
+
+void bs_remap_earlyboot(void);
+
 static __inline void *
 __ppc_ba(bus_space_handle_t bsh, bus_size_t ofs)
 {
@@ -58,10 +71,44 @@ static int
 bs_gen_map(bus_addr_t addr, bus_size_t size __unused, int flags __unused,
     bus_space_handle_t *bshp)
 {
-	*bshp = addr;
+	/*
+	 * Record what we did if we haven't enabled the MMU yet. We
+	 * will need to remap it as soon as the MMU comes up.
+	 */
+	if (!pmap_bootstrapped) {
+		KASSERT(earlyboot_map_idx < MAX_EARLYBOOT_MAPPINGS,
+		    ("%s: too many early boot mapping requests", __func__));
+		earlyboot_mappings[earlyboot_map_idx].addr = addr;
+		earlyboot_mappings[earlyboot_map_idx].size = size;
+		earlyboot_map_idx++;
+		*bshp = addr;
+	} else {
+		*bshp = (bus_space_handle_t)pmap_mapdev(addr,size);
+	}
+
 	return (0);
 }
 
+void
+bs_remap_earlyboot(void)
+{
+	int i;
+	vm_offset_t pa, spa;
+
+	if (hw_direct_map)
+		return;
+
+	for (i = 0; i < earlyboot_map_idx; i++) {
+		spa = earlyboot_mappings[i].addr;
+
+		pa = trunc_page(spa);
+		while (pa < spa + earlyboot_mappings[i].size) {
+			pmap_kenter(pa,pa);
+			pa += PAGE_SIZE;
+		}
+	}
+}
+
 static void
 bs_gen_unmap(bus_size_t size __unused)
 {
diff --git a/sys/powerpc/powerpc/cpu.c b/sys/powerpc/powerpc/cpu.c
index 4724313..a15ef58 100644
--- a/sys/powerpc/powerpc/cpu.c
+++ b/sys/powerpc/powerpc/cpu.c
@@ -91,6 +91,10 @@ static const struct cputab models[] = {
         { "Motorola PowerPC 620",	MPC620,		REVFMT_HEX },
         { "Motorola PowerPC 750",	MPC750,		REVFMT_MAJMIN },
         { "IBM PowerPC 750FX",		IBM750FX,	REVFMT_MAJMIN },
+        { "IBM PowerPC 970",		IBM970,		REVFMT_MAJMIN },
+        { "IBM PowerPC 970FX",		IBM970FX,	REVFMT_MAJMIN },
+        { "IBM PowerPC 970GX",		IBM970GX,	REVFMT_MAJMIN },
+        { "IBM PowerPC 970MP",		IBM970MP,	REVFMT_MAJMIN },
         { "Motorola PowerPC 7400",	MPC7400,	REVFMT_MAJMIN },
         { "Motorola PowerPC 7410",	MPC7410,	REVFMT_MAJMIN },
         { "Motorola PowerPC 7450",	MPC7450,	REVFMT_MAJMIN },
diff --git a/sys/powerpc/powerpc/mem.c b/sys/powerpc/powerpc/mem.c
index b3c320b..22d99c4 100644
--- a/sys/powerpc/powerpc/mem.c
+++ b/sys/powerpc/powerpc/mem.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #include <vm/vm_extern.h>
+#include <vm/vm_page.h>
 
 #include <machine/memdev.h>
 
@@ -77,6 +78,8 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	int error = 0;
 	vm_offset_t va, eva, off, v;
 	vm_prot_t prot;
+	struct vm_page m;
+	vm_page_t marr;
 	vm_size_t cnt;
 
 	cnt = 0;
@@ -102,14 +105,18 @@ kmem_direct_mapped:	v = uio->uio_offset;
 			cnt = min(cnt, PAGE_SIZE - off);
 			cnt = min(cnt, iov->iov_len);
 
-			if (mem_valid(v, cnt)
-			    && pmap_dev_direct_mapped(v, cnt)) {
+			if (mem_valid(v, cnt)) {
 				error = EFAULT;
 				break;
 			}
-
-			uiomove((void *)v, cnt, uio);
-			break;
+	
+			if (!pmap_dev_direct_mapped(v, cnt)) {
+				error = uiomove((void *)v, cnt, uio);
+			} else {
+				m.phys_addr = trunc_page(v);
+				marr = &m;
+				error = uiomove_fromphys(&marr, off, cnt, uio);
+			}
 		}
 		else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
 			va = uio->uio_offset;
diff --git a/sys/powerpc/powerpc/mmu_if.m b/sys/powerpc/powerpc/mmu_if.m
index 4a8ffa8..9b7bdd2 100644
--- a/sys/powerpc/powerpc/mmu_if.m
+++ b/sys/powerpc/powerpc/mmu_if.m
@@ -697,6 +697,18 @@ METHOD void bootstrap {
 	vm_offset_t	_end;
 };
 
+/**
+ * @brief Set up the MMU on the current CPU. Only called by the PMAP layer
+ * for alternate CPUs on SMP systems.
+ *
+ * @param _ap		Set to 1 if the CPU being set up is an AP
+ *
+ */
+METHOD void cpu_bootstrap {
+	mmu_t		_mmu;
+	int		_ap;
+};
+
 
 /**
  * @brief Create a kernel mapping for a given physical address range.
diff --git a/sys/powerpc/powerpc/pmap_dispatch.c b/sys/powerpc/powerpc/pmap_dispatch.c
index 33e94d1..a60c076 100644
--- a/sys/powerpc/powerpc/pmap_dispatch.c
+++ b/sys/powerpc/powerpc/pmap_dispatch.c
@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_page.h>
 
 #include <machine/mmuvar.h>
+#include <machine/smp.h>
 
 #include "mmu_if.h"
 
@@ -406,6 +407,16 @@ pmap_bootstrap(vm_offset_t start, vm_offset_t end)
 	MMU_BOOTSTRAP(mmu_obj, start, end);
 }
 
+void
+pmap_cpu_bootstrap(int ap)
+{
+	/*
+	 * No KTR here because our console probably doesn't work yet
+	 */
+
+	return (MMU_CPU_BOOTSTRAP(mmu_obj, ap));
+}
+
 void *
 pmap_mapdev(vm_offset_t pa, vm_size_t size)
 {
diff --git a/sys/powerpc/booke/uio_machdep.c b/sys/powerpc/powerpc/uio_machdep.c
index 2a88fd2..2a88fd2 100644
--- a/sys/powerpc/booke/uio_machdep.c
+++ b/sys/powerpc/powerpc/uio_machdep.c