summaryrefslogtreecommitdiffstats
path: root/sys/powerpc/aim/mmu_oea64.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/powerpc/aim/mmu_oea64.c')
-rw-r--r--sys/powerpc/aim/mmu_oea64.c955
1 files changed, 699 insertions, 256 deletions
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 8672ab2..33b3ed9 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -142,12 +142,14 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_pager.h>
#include <vm/uma.h>
+#include <machine/_inttypes.h>
#include <machine/cpu.h>
#include <machine/platform.h>
#include <machine/frame.h>
#include <machine/md_var.h>
#include <machine/psl.h>
#include <machine/bat.h>
+#include <machine/hid.h>
#include <machine/pte.h>
#include <machine/sr.h>
#include <machine/trap.h>
@@ -158,20 +160,16 @@ __FBSDID("$FreeBSD$");
#define MOEA_DEBUG
#define TODO panic("%s: not implemented", __func__);
+void moea64_release_vsid(uint64_t vsid);
+uintptr_t moea64_get_unique_vsid(void);
-static __inline u_int32_t
-cntlzw(volatile u_int32_t a) {
- u_int32_t b;
- __asm ("cntlzw %0, %1" : "=r"(b) : "r"(a));
+static __inline register_t
+cntlzd(volatile register_t a) {
+ register_t b;
+ __asm ("cntlzd %0, %1" : "=r"(b) : "r"(a));
return b;
}
-static __inline uint64_t
-va_to_vsid(pmap_t pm, vm_offset_t va)
-{
- return ((pm->pm_sr[(uintptr_t)va >> ADDR_SR_SHFT]) & SR_VSID_MASK);
-}
-
#define PTESYNC() __asm __volatile("ptesync");
#define TLBSYNC() __asm __volatile("tlbsync; ptesync");
#define SYNC() __asm __volatile("sync");
@@ -186,21 +184,29 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
struct mtx tlbie_mutex;
static __inline void
-TLBIE(pmap_t pmap, vm_offset_t va) {
- uint64_t vpn;
+TLBIE(uint64_t vpn) {
+#ifndef __powerpc64__
register_t vpn_hi, vpn_lo;
register_t msr;
register_t scratch;
+#endif
- vpn = (uint64_t)(va & ADDR_PIDX);
- if (pmap != NULL)
- vpn |= (va_to_vsid(pmap,va) << 28);
+ vpn <<= ADDR_PIDX_SHFT;
vpn &= ~(0xffffULL << 48);
+ mtx_lock_spin(&tlbie_mutex);
+#ifdef __powerpc64__
+ __asm __volatile("\
+ ptesync; \
+ tlbie %0; \
+ eieio; \
+ tlbsync; \
+ ptesync;"
+ :: "r"(vpn) : "memory");
+#else
vpn_hi = (uint32_t)(vpn >> 32);
vpn_lo = (uint32_t)vpn;
- mtx_lock_spin(&tlbie_mutex);
__asm __volatile("\
mfmsr %0; \
mr %1, %0; \
@@ -218,6 +224,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
ptesync;"
: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
: "memory");
+#endif
mtx_unlock_spin(&tlbie_mutex);
}
@@ -225,7 +232,6 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define ENABLE_TRANS(msr) mtmsr(msr); isync()
#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4))
-#define VSID_TO_SR(vsid) ((vsid) & 0xf)
#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
#define VSID_HASH_MASK 0x0000007fffffffffULL
@@ -236,6 +242,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define PVO_BOOTSTRAP 0x080UL /* PVO entry allocated during
bootstrap */
#define PVO_FAKE 0x100UL /* fictitious phys page */
+#define PVO_LARGE 0x200UL /* large page */
#define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
#define PVO_ISFAKE(pvo) ((pvo)->pvo_vaddr & PVO_FAKE)
#define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
@@ -244,6 +251,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
#define PVO_PTEGIDX_SET(pvo, i) \
((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
+#define PVO_VSID(pvo) ((pvo)->pvo_vpn >> 16)
#define MOEA_PVO_CHECK(pvo)
@@ -252,11 +260,11 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define ASSERT_TABLE_LOCK() mtx_assert(&moea64_table_mutex, MA_OWNED)
struct ofw_map {
- vm_offset_t om_va;
- vm_size_t om_len;
- vm_offset_t om_pa_hi;
- vm_offset_t om_pa_lo;
- u_int om_mode;
+ cell_t om_va;
+ cell_t om_len;
+ cell_t om_pa_hi;
+ cell_t om_pa_lo;
+ cell_t om_mode;
};
/*
@@ -264,8 +272,8 @@ struct ofw_map {
*/
static struct mem_region *regions;
static struct mem_region *pregions;
-extern u_int phys_avail_count;
-extern int regions_sz, pregions_sz;
+static u_int phys_avail_count;
+static int regions_sz, pregions_sz;
extern int ofw_real_mode;
extern struct pmap ofw_pmap;
@@ -303,7 +311,14 @@ static struct pvo_entry *moea64_bpvo_pool;
static int moea64_bpvo_pool_index = 0;
#define VSID_NBPW (sizeof(u_int32_t) * 8)
-static u_int moea64_vsid_bitmap[NPMAPS / VSID_NBPW];
+#ifdef __powerpc64__
+#define NVSIDS (NPMAPS * 16)
+#define VSID_HASHMASK 0xffffffffUL
+#else
+#define NVSIDS NPMAPS
+#define VSID_HASHMASK 0xfffffUL
+#endif
+static u_int moea64_vsid_bitmap[NVSIDS / VSID_NBPW];
static boolean_t moea64_initialized = FALSE;
@@ -327,9 +342,14 @@ SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD,
&moea64_pvo_remove_calls, 0, "");
vm_offset_t moea64_scratchpage_va[2];
+uint64_t moea64_scratchpage_vpn[2];
struct lpte *moea64_scratchpage_pte[2];
struct mtx moea64_scratchpage_mtx;
+uint64_t moea64_large_page_mask = 0;
+int moea64_large_page_size = 0;
+int moea64_large_page_shift = 0;
+
/*
* Allocate physical memory for use in moea64_bootstrap.
*/
@@ -345,16 +365,16 @@ static int moea64_pte_insert(u_int, struct lpte *);
*/
static int moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *,
vm_offset_t, vm_offset_t, uint64_t, int);
-static void moea64_pvo_remove(struct pvo_entry *, int);
-static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t, int *);
-static struct lpte *moea64_pvo_to_pte(const struct pvo_entry *, int);
+static void moea64_pvo_remove(struct pvo_entry *);
+static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t);
+static struct lpte *moea64_pvo_to_pte(const struct pvo_entry *);
/*
* Utility routines.
*/
-static void moea64_bridge_bootstrap(mmu_t mmup,
+static void moea64_bootstrap(mmu_t mmup,
vm_offset_t kernelstart, vm_offset_t kernelend);
-static void moea64_bridge_cpu_bootstrap(mmu_t, int ap);
+static void moea64_cpu_bootstrap(mmu_t, int ap);
static void moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t,
vm_prot_t, boolean_t);
static boolean_t moea64_query_bit(vm_page_t, u_int64_t);
@@ -363,6 +383,9 @@ static void moea64_kremove(mmu_t, vm_offset_t);
static void moea64_syncicache(pmap_t pmap, vm_offset_t va,
vm_offset_t pa, vm_size_t sz);
static void tlbia(void);
+#ifdef __powerpc64__
+static void slbia(void);
+#endif
/*
* Kernel MMU interface
@@ -405,7 +428,7 @@ void moea64_kenter(mmu_t, vm_offset_t, vm_offset_t);
boolean_t moea64_dev_direct_mapped(mmu_t, vm_offset_t, vm_size_t);
static void moea64_sync_icache(mmu_t, pmap_t, vm_offset_t, vm_size_t);
-static mmu_method_t moea64_bridge_methods[] = {
+static mmu_method_t moea64_methods[] = {
MMUMETHOD(mmu_change_wiring, moea64_change_wiring),
MMUMETHOD(mmu_clear_modify, moea64_clear_modify),
MMUMETHOD(mmu_clear_reference, moea64_clear_reference),
@@ -439,8 +462,8 @@ static mmu_method_t moea64_bridge_methods[] = {
MMUMETHOD(mmu_deactivate, moea64_deactivate),
/* Internal interfaces */
- MMUMETHOD(mmu_bootstrap, moea64_bridge_bootstrap),
- MMUMETHOD(mmu_cpu_bootstrap, moea64_bridge_cpu_bootstrap),
+ MMUMETHOD(mmu_bootstrap, moea64_bootstrap),
+ MMUMETHOD(mmu_cpu_bootstrap, moea64_cpu_bootstrap),
MMUMETHOD(mmu_mapdev, moea64_mapdev),
MMUMETHOD(mmu_unmapdev, moea64_unmapdev),
MMUMETHOD(mmu_kextract, moea64_kextract),
@@ -450,20 +473,22 @@ static mmu_method_t moea64_bridge_methods[] = {
{ 0, 0 }
};
-static mmu_def_t oea64_bridge_mmu = {
+static mmu_def_t oea64_mmu = {
MMU_TYPE_G5,
- moea64_bridge_methods,
+ moea64_methods,
0
};
-MMU_DEF(oea64_bridge_mmu);
+MMU_DEF(oea64_mmu);
static __inline u_int
-va_to_pteg(uint64_t vsid, vm_offset_t addr)
+va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
{
uint64_t hash;
+ int shift;
+ shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT;
hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >>
- ADDR_PIDX_SHFT);
+ shift);
return (hash & moea64_pteg_mask);
}
@@ -515,8 +540,9 @@ moea64_attr_save(vm_page_t m, u_int64_t ptebit)
static __inline void
moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
- uint64_t pte_lo)
+ uint64_t pte_lo, int flags)
{
+
ASSERT_TABLE_LOCK();
/*
@@ -528,6 +554,9 @@ moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
pt->pte_hi = (vsid << LPTE_VSID_SHIFT) |
(((uint64_t)(va & ADDR_PIDX) >> ADDR_API_SHFT64) & LPTE_API);
+ if (flags & PVO_LARGE)
+ pt->pte_hi |= LPTE_BIG;
+
pt->pte_lo = pte_lo;
}
@@ -541,7 +570,7 @@ moea64_pte_synch(struct lpte *pt, struct lpte *pvo_pt)
}
static __inline void
-moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit)
+moea64_pte_clear(struct lpte *pt, uint64_t vpn, u_int64_t ptebit)
{
ASSERT_TABLE_LOCK();
@@ -549,7 +578,7 @@ moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit)
* As shown in Section 7.6.3.2.3
*/
pt->pte_lo &= ~ptebit;
- TLBIE(pmap,va);
+ TLBIE(vpn);
}
static __inline void
@@ -572,7 +601,7 @@ moea64_pte_set(struct lpte *pt, struct lpte *pvo_pt)
}
static __inline void
-moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn)
{
ASSERT_TABLE_LOCK();
pvo_pt->pte_hi &= ~LPTE_VALID;
@@ -586,7 +615,7 @@ moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t
* Invalidate the pte.
*/
pt->pte_hi &= ~LPTE_VALID;
- TLBIE(pmap,va);
+ TLBIE(vpn);
/*
* Save the reg & chg bits.
@@ -596,16 +625,14 @@ moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t
}
static __inline void
-moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn)
{
/*
* Invalidate the PTE
*/
- moea64_pte_unset(pt, pvo_pt, pmap, va);
+ moea64_pte_unset(pt, pvo_pt, vpn);
moea64_pte_set(pt, pvo_pt);
- if (pmap == kernel_pmap)
- isync();
}
static __inline uint64_t
@@ -674,21 +701,45 @@ om_cmp(const void *a, const void *b)
}
static void
-moea64_bridge_cpu_bootstrap(mmu_t mmup, int ap)
+moea64_cpu_bootstrap(mmu_t mmup, int ap)
{
int i = 0;
+ #ifdef __powerpc64__
+ struct slb *slb = PCPU_GET(slb);
+ #endif
/*
* Initialize segment registers and MMU
*/
mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync();
- for (i = 0; i < 16; i++) {
- mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
- }
+
+ /*
+ * Install kernel SLB entries
+ */
+
+ #ifdef __powerpc64__
+ slbia();
+
+ for (i = 0; i < 64; i++) {
+ if (!(slb[i].slbe & SLBE_VALID))
+ continue;
+
+ __asm __volatile ("slbmte %0, %1" ::
+ "r"(slb[i].slbv), "r"(slb[i].slbe));
+ }
+ #else
+ for (i = 0; i < 16; i++)
+ mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
+ #endif
+
+ /*
+ * Install page table
+ */
+
__asm __volatile ("ptesync; mtsdr1 %0; isync"
- :: "r"((u_int)moea64_pteg_table
- | (32 - cntlzw(moea64_pteg_mask >> 11))));
+ :: "r"((uintptr_t)moea64_pteg_table
+ | (64 - cntlzd(moea64_pteg_mask >> 11))));
tlbia();
}
@@ -717,15 +768,23 @@ moea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz)
if (translations[i].om_pa_lo % PAGE_SIZE)
panic("OFW translation not page-aligned!");
+ pa_base = translations[i].om_pa_lo;
+
+ #ifdef __powerpc64__
+ pa_base += (vm_offset_t)translations[i].om_pa_hi << 32;
+ #else
if (translations[i].om_pa_hi)
panic("OFW translations above 32-bit boundary!");
-
- pa_base = translations[i].om_pa_lo;
+ #endif
/* Now enter the pages for this mapping */
DISABLE_TRANS(msr);
for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) {
+ if (moea64_pvo_find_va(kernel_pmap,
+ translations[i].om_va + off) != NULL)
+ continue;
+
moea64_kenter(mmup, translations[i].om_va + off,
pa_base + off);
@@ -735,8 +794,130 @@ moea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz)
}
}
+#ifdef __powerpc64__
+static void
+moea64_probe_large_page(void)
+{
+ uint16_t pvr = mfpvr() >> 16;
+
+ switch (pvr) {
+ case IBM970:
+ case IBM970FX:
+ case IBM970MP:
+ powerpc_sync(); isync();
+ mtspr(SPR_HID4, mfspr(SPR_HID4) & ~HID4_970_DISABLE_LG_PG);
+ powerpc_sync(); isync();
+
+ /* FALLTHROUGH */
+ case IBMCELLBE:
+ moea64_large_page_size = 0x1000000; /* 16 MB */
+ moea64_large_page_shift = 24;
+ break;
+ default:
+ moea64_large_page_size = 0;
+ }
+
+ moea64_large_page_mask = moea64_large_page_size - 1;
+}
+
+static void
+moea64_bootstrap_slb_prefault(vm_offset_t va, int large)
+{
+ struct slb *cache;
+ struct slb entry;
+ uint64_t esid, slbe;
+ uint64_t i;
+
+ cache = PCPU_GET(slb);
+ esid = va >> ADDR_SR_SHFT;
+ slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
+
+ for (i = 0; i < 64; i++) {
+ if (cache[i].slbe == (slbe | i))
+ return;
+ }
+
+ entry.slbe = slbe;
+ entry.slbv = KERNEL_VSID(esid, large) << SLBV_VSID_SHIFT;
+ if (large)
+ entry.slbv |= SLBV_L;
+
+ slb_insert(kernel_pmap, cache, &entry);
+}
+#endif
+
+static void
+moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart,
+ vm_offset_t kernelend)
+{
+ register_t msr;
+ vm_paddr_t pa;
+ vm_offset_t size, off;
+ uint64_t pte_lo;
+ int i;
+
+ if (moea64_large_page_size == 0)
+ hw_direct_map = 0;
+
+ DISABLE_TRANS(msr);
+ if (hw_direct_map) {
+ PMAP_LOCK(kernel_pmap);
+ for (i = 0; i < pregions_sz; i++) {
+ for (pa = pregions[i].mr_start; pa < pregions[i].mr_start +
+ pregions[i].mr_size; pa += moea64_large_page_size) {
+ pte_lo = LPTE_M;
+
+ /*
+ * Set memory access as guarded if prefetch within
+ * the page could exit the available physmem area.
+ */
+ if (pa & moea64_large_page_mask) {
+ pa &= moea64_large_page_mask;
+ pte_lo |= LPTE_G;
+ }
+ if (pa + moea64_large_page_size >
+ pregions[i].mr_start + pregions[i].mr_size)
+ pte_lo |= LPTE_G;
+
+ moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+ &moea64_pvo_kunmanaged, pa, pa,
+ pte_lo, PVO_WIRED | PVO_LARGE |
+ VM_PROT_EXECUTE);
+ }
+ }
+ PMAP_UNLOCK(kernel_pmap);
+ } else {
+ size = moea64_pteg_count * sizeof(struct lpteg);
+ off = (vm_offset_t)(moea64_pteg_table);
+ for (pa = off; pa < off + size; pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+ size = sizeof(struct pvo_head) * moea64_pteg_count;
+ off = (vm_offset_t)(moea64_pvo_table);
+ for (pa = off; pa < off + size; pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+ size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
+ off = (vm_offset_t)(moea64_bpvo_pool);
+ for (pa = off; pa < off + size; pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+
+ /*
+ * Map certain important things, like ourselves.
+ *
+ * NOTE: We do not map the exception vector space. That code is
+ * used only in real mode, and leaving it unmapped allows us to
+ * catch NULL pointer deferences, instead of making NULL a valid
+ * address.
+ */
+
+ for (pa = kernelstart & ~PAGE_MASK; pa < kernelend;
+ pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+ }
+ ENABLE_TRANS(msr);
+}
+
static void
-moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
{
ihandle_t mmui;
phandle_t chosen;
@@ -744,10 +925,11 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
size_t sz;
int i, j;
vm_size_t size, physsz, hwphyssz;
- vm_offset_t pa, va, off;
+ vm_offset_t pa, va;
register_t msr;
void *dpcpu;
+#ifndef __powerpc64__
/* We don't have a direct map since there is no BAT */
hw_direct_map = 0;
@@ -756,6 +938,15 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
battable[i].batu = 0;
battable[i].batl = 0;
}
+#else
+ moea64_probe_large_page();
+
+ /* Use a direct map if we have large page support */
+ if (moea64_large_page_size > 0)
+ hw_direct_map = 1;
+ else
+ hw_direct_map = 0;
+#endif
/* Get physical memory regions from firmware */
mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
@@ -789,6 +980,38 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
phys_avail_count++;
physsz += regions[i].mr_size;
}
+
+ /* Check for overlap with the kernel and exception vectors */
+ for (j = 0; j < 2*phys_avail_count; j+=2) {
+ if (phys_avail[j] < EXC_LAST)
+ phys_avail[j] += EXC_LAST;
+
+ if (kernelstart >= phys_avail[j] &&
+ kernelstart < phys_avail[j+1]) {
+ if (kernelend < phys_avail[j+1]) {
+ phys_avail[2*phys_avail_count] =
+ (kernelend & ~PAGE_MASK) + PAGE_SIZE;
+ phys_avail[2*phys_avail_count + 1] =
+ phys_avail[j+1];
+ phys_avail_count++;
+ }
+
+ phys_avail[j+1] = kernelstart & ~PAGE_MASK;
+ }
+
+ if (kernelend >= phys_avail[j] &&
+ kernelend < phys_avail[j+1]) {
+ if (kernelstart > phys_avail[j]) {
+ phys_avail[2*phys_avail_count] = phys_avail[j];
+ phys_avail[2*phys_avail_count + 1] =
+ kernelstart & ~PAGE_MASK;
+ phys_avail_count++;
+ }
+
+ phys_avail[j] = (kernelend & ~PAGE_MASK) + PAGE_SIZE;
+ }
+ }
+
physmem = btoc(physsz);
/*
@@ -801,6 +1024,8 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
while (moea64_pteg_count < physmem)
moea64_pteg_count <<= 1;
+
+ moea64_pteg_count >>= 1;
#endif /* PTEGCOUNT */
size = moea64_pteg_count * sizeof(struct lpteg);
@@ -859,15 +1084,24 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
/*
* Make sure kernel vsid is allocated as well as VSID 0.
*/
- moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW]
+ #ifndef __powerpc64__
+ moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NVSIDS - 1)) / VSID_NBPW]
|= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
moea64_vsid_bitmap[0] |= 1;
+ #endif
/*
* Initialize the kernel pmap (which is statically allocated).
*/
+ #ifdef __powerpc64__
+ for (i = 0; i < 64; i++) {
+ pcpup->pc_slb[i].slbv = 0;
+ pcpup->pc_slb[i].slbe = 0;
+ }
+ #else
for (i = 0; i < 16; i++)
kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i;
+ #endif
kernel_pmap->pmap_phys = kernel_pmap;
kernel_pmap->pm_active = ~0;
@@ -878,45 +1112,25 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
* Now map in all the other buffers we allocated earlier
*/
- DISABLE_TRANS(msr);
- size = moea64_pteg_count * sizeof(struct lpteg);
- off = (vm_offset_t)(moea64_pteg_table);
- for (pa = off; pa < off + size; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
- size = sizeof(struct pvo_head) * moea64_pteg_count;
- off = (vm_offset_t)(moea64_pvo_table);
- for (pa = off; pa < off + size; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
- size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
- off = (vm_offset_t)(moea64_bpvo_pool);
- for (pa = off; pa < off + size; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
+ moea64_setup_direct_map(mmup, kernelstart, kernelend);
/*
- * Map certain important things, like ourselves.
- *
- * NOTE: We do not map the exception vector space. That code is
- * used only in real mode, and leaving it unmapped allows us to
- * catch NULL pointer deferences, instead of making NULL a valid
- * address.
+ * Set up the Open Firmware pmap and add its mappings if not in real
+ * mode.
*/
- for (pa = kernelstart & ~PAGE_MASK; pa < kernelend; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
- ENABLE_TRANS(msr);
-
if (!ofw_real_mode) {
- /*
- * Set up the Open Firmware pmap and add its mappings.
- */
-
+ #ifndef __powerpc64__
moea64_pinit(mmup, &ofw_pmap);
+
for (i = 0; i < 16; i++)
ofw_pmap.pm_sr[i] = kernel_pmap->pm_sr[i];
+ #endif
if ((chosen = OF_finddevice("/chosen")) == -1)
panic("moea64_bootstrap: can't find /chosen");
OF_getprop(chosen, "mmu", &mmui, 4);
+
if ((mmu = OF_instance_to_package(mmui)) == -1)
panic("moea64_bootstrap: can't get mmu package");
if ((sz = OF_getproplen(mmu, "translations")) == -1)
@@ -941,7 +1155,7 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
/*
* Initialize MMU and remap early physical mappings
*/
- moea64_bridge_cpu_bootstrap(mmup,0);
+ moea64_cpu_bootstrap(mmup,0);
mtmsr(mfmsr() | PSL_DR | PSL_IR); isync();
pmap_bootstrapped++;
bs_remap_earlyboot();
@@ -953,6 +1167,14 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS;
/*
+ * Map the entire KVA range into the SLB. We must not fault there.
+ */
+ #ifdef __powerpc64__
+ for (va = virtual_avail; va < virtual_end; va += SEGMENT_LENGTH)
+ moea64_bootstrap_slb_prefault(va, 0);
+ #endif
+
+ /*
* Figure out how far we can extend virtual_end into segment 16
* without running into existing mappings. Segment 16 is guaranteed
* to contain neither RAM nor devices (at least on Apple hardware),
@@ -960,10 +1182,13 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
* step on.
*/
+ #ifndef __powerpc64__ /* KVA is in high memory on PPC64 */
PMAP_LOCK(kernel_pmap);
- while (moea64_pvo_find_va(kernel_pmap, virtual_end+1, NULL) == NULL)
+ while (virtual_end < VM_MAX_KERNEL_ADDRESS &&
+ moea64_pvo_find_va(kernel_pmap, virtual_end+1) == NULL)
virtual_end += PAGE_SIZE;
PMAP_UNLOCK(kernel_pmap);
+ #endif
/*
* Allocate some things for page zeroing. We put this directly
@@ -972,31 +1197,38 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
* from even knowing that this hack exists.
*/
- mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL, MTX_DEF);
- for (i = 0; i < 2; i++) {
- struct lpte pt;
- uint64_t vsid;
- int pteidx, ptegidx;
-
- moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
- virtual_end -= PAGE_SIZE;
-
- LOCK_TABLE();
-
- vsid = va_to_vsid(kernel_pmap, moea64_scratchpage_va[i]);
- moea64_pte_create(&pt, vsid, moea64_scratchpage_va[i],
- LPTE_NOEXEC);
- pt.pte_hi |= LPTE_LOCKED;
-
- ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i]);
- pteidx = moea64_pte_insert(ptegidx, &pt);
- if (pt.pte_hi & LPTE_HID)
- ptegidx ^= moea64_pteg_mask;
-
- moea64_scratchpage_pte[i] =
- &moea64_pteg_table[ptegidx].pt[pteidx];
-
- UNLOCK_TABLE();
+ if (!hw_direct_map) {
+ mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL,
+ MTX_DEF);
+ for (i = 0; i < 2; i++) {
+ struct lpte pt;
+ uint64_t vsid;
+ int pteidx, ptegidx;
+
+ moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
+ virtual_end -= PAGE_SIZE;
+
+ LOCK_TABLE();
+
+ vsid = va_to_vsid(kernel_pmap,
+ moea64_scratchpage_va[i]);
+ moea64_pte_create(&pt, vsid, moea64_scratchpage_va[i],
+ LPTE_NOEXEC, 0);
+ pt.pte_hi |= LPTE_LOCKED;
+
+ moea64_scratchpage_vpn[i] = (vsid << 16) |
+ ((moea64_scratchpage_va[i] & ADDR_PIDX) >>
+ ADDR_PIDX_SHFT);
+ ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i], 0);
+ pteidx = moea64_pte_insert(ptegidx, &pt);
+ if (pt.pte_hi & LPTE_HID)
+ ptegidx ^= moea64_pteg_mask;
+
+ moea64_scratchpage_pte[i] =
+ &moea64_pteg_table[ptegidx].pt[pteidx];
+
+ UNLOCK_TABLE();
+ }
}
/*
@@ -1033,6 +1265,7 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
*/
pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE);
dpcpu = (void *)virtual_avail;
+ va = virtual_avail;
virtual_avail += DPCPU_SIZE;
while (va < virtual_avail) {
moea64_kenter(mmup, va, pa);
@@ -1043,23 +1276,22 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
}
/*
- * Activate a user pmap. The pmap must be activated before it's address
+ * Activate a user pmap. The pmap must be activated before its address
* space can be accessed in any way.
*/
void
moea64_activate(mmu_t mmu, struct thread *td)
{
- pmap_t pm, pmr;
+ pmap_t pm;
- /*
- * Load all the data we need up front to encourage the compiler to
- * not issue any loads while we have interrupts disabled below.
- */
pm = &td->td_proc->p_vmspace->vm_pmap;
- pmr = pm->pmap_phys;
-
pm->pm_active |= PCPU_GET(cpumask);
- PCPU_SET(curpmap, pmr);
+
+ #ifdef __powerpc64__
+ PCPU_SET(userslb, pm->pm_slb);
+ #else
+ PCPU_SET(curpmap, pm->pmap_phys);
+ #endif
}
void
@@ -1069,27 +1301,61 @@ moea64_deactivate(mmu_t mmu, struct thread *td)
pm = &td->td_proc->p_vmspace->vm_pmap;
pm->pm_active &= ~(PCPU_GET(cpumask));
+ #ifdef __powerpc64__
+ PCPU_SET(userslb, NULL);
+ #else
PCPU_SET(curpmap, NULL);
+ #endif
}
void
moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
{
struct pvo_entry *pvo;
+ struct lpte *pt;
+ uint64_t vsid;
+ int i, ptegidx;
PMAP_LOCK(pm);
- pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
if (pvo != NULL) {
+ LOCK_TABLE();
+ pt = moea64_pvo_to_pte(pvo);
+
if (wired) {
if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
pm->pm_stats.wired_count++;
pvo->pvo_vaddr |= PVO_WIRED;
+ pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED;
} else {
if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
pm->pm_stats.wired_count--;
pvo->pvo_vaddr &= ~PVO_WIRED;
+ pvo->pvo_pte.lpte.pte_hi &= ~LPTE_WIRED;
+ }
+
+ if (pt != NULL) {
+ /* Update wiring flag in page table. */
+ moea64_pte_change(pt, &pvo->pvo_pte.lpte,
+ pvo->pvo_vpn);
+ } else if (wired) {
+ /*
+ * If we are wiring the page, and it wasn't in the
+ * page table before, add it.
+ */
+ vsid = PVO_VSID(pvo);
+ ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
+ pvo->pvo_vaddr & PVO_LARGE);
+
+ i = moea64_pte_insert(ptegidx, &pvo->pvo_pte.lpte);
+ if (i >= 0) {
+ PVO_PTEGIDX_CLR(pvo);
+ PVO_PTEGIDX_SET(pvo, i);
+ }
}
+
+ UNLOCK_TABLE();
}
PMAP_UNLOCK(pm);
}
@@ -1104,10 +1370,11 @@ moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
static __inline
void moea64_set_scratchpage_pa(int which, vm_offset_t pa) {
+ KASSERT(!hw_direct_map, ("Using OEA64 scratchpage with a direct map!"));
mtx_assert(&moea64_scratchpage_mtx, MA_OWNED);
moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID;
- TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+ TLBIE(moea64_scratchpage_vpn[which]);
moea64_scratchpage_pte[which]->pte_lo &=
~(LPTE_WIMG | LPTE_RPGN);
@@ -1128,15 +1395,19 @@ moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst)
dst = VM_PAGE_TO_PHYS(mdst);
src = VM_PAGE_TO_PHYS(msrc);
- mtx_lock(&moea64_scratchpage_mtx);
+ if (hw_direct_map) {
+ kcopy((void *)src, (void *)dst, PAGE_SIZE);
+ } else {
+ mtx_lock(&moea64_scratchpage_mtx);
- moea64_set_scratchpage_pa(0,src);
- moea64_set_scratchpage_pa(1,dst);
+ moea64_set_scratchpage_pa(0,src);
+ moea64_set_scratchpage_pa(1,dst);
- kcopy((void *)moea64_scratchpage_va[0],
- (void *)moea64_scratchpage_va[1], PAGE_SIZE);
+ kcopy((void *)moea64_scratchpage_va[0],
+ (void *)moea64_scratchpage_va[1], PAGE_SIZE);
- mtx_unlock(&moea64_scratchpage_mtx);
+ mtx_unlock(&moea64_scratchpage_mtx);
+ }
}
void
@@ -1145,15 +1416,18 @@ moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
vm_offset_t pa = VM_PAGE_TO_PHYS(m);
if (!moea64_initialized)
- panic("moea64_zero_page: can't zero pa %#x", pa);
+ panic("moea64_zero_page: can't zero pa %#" PRIxPTR, pa);
if (size + off > PAGE_SIZE)
panic("moea64_zero_page: size + off > PAGE_SIZE");
- mtx_lock(&moea64_scratchpage_mtx);
-
- moea64_set_scratchpage_pa(0,pa);
- bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
- mtx_unlock(&moea64_scratchpage_mtx);
+ if (hw_direct_map) {
+ bzero((caddr_t)pa + off, size);
+ } else {
+ mtx_lock(&moea64_scratchpage_mtx);
+ moea64_set_scratchpage_pa(0,pa);
+ bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
+ mtx_unlock(&moea64_scratchpage_mtx);
+ }
}
/*
@@ -1163,18 +1437,25 @@ void
moea64_zero_page(mmu_t mmu, vm_page_t m)
{
vm_offset_t pa = VM_PAGE_TO_PHYS(m);
- vm_offset_t off;
+ vm_offset_t va, off;
if (!moea64_initialized)
- panic("moea64_zero_page: can't zero pa %#x", pa);
+ panic("moea64_zero_page: can't zero pa %#zx", pa);
+
+ if (!hw_direct_map) {
+ mtx_lock(&moea64_scratchpage_mtx);
- mtx_lock(&moea64_scratchpage_mtx);
+ moea64_set_scratchpage_pa(0,pa);
+ va = moea64_scratchpage_va[0];
+ } else {
+ va = pa;
+ }
- moea64_set_scratchpage_pa(0,pa);
for (off = 0; off < PAGE_SIZE; off += cacheline_size)
- __asm __volatile("dcbz 0,%0" ::
- "r"(moea64_scratchpage_va[0] + off));
- mtx_unlock(&moea64_scratchpage_mtx);
+ __asm __volatile("dcbz 0,%0" :: "r"(va + off));
+
+ if (!hw_direct_map)
+ mtx_unlock(&moea64_scratchpage_mtx);
}
void
@@ -1298,6 +1579,8 @@ moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz)
__syncicache((void *)pa, sz);
} else if (pmap == kernel_pmap) {
__syncicache((void *)va, sz);
+ } else if (hw_direct_map) {
+ __syncicache((void *)pa, sz);
} else {
/* Use the scratch page to set up a temp mapping */
@@ -1363,11 +1646,12 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va)
vm_paddr_t pa;
PMAP_LOCK(pm);
- pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pm, va);
if (pvo == NULL)
pa = 0;
else
- pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+ pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) |
+ (va - PVO_VADDR(pvo));
PMAP_UNLOCK(pm);
return (pa);
}
@@ -1388,7 +1672,7 @@ moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot)
pa = 0;
PMAP_LOCK(pmap);
retry:
- pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW ||
(prot & VM_PROT_WRITE) == 0)) {
@@ -1565,7 +1849,7 @@ moea64_remove_write(mmu_t mmu, vm_page_t m)
PMAP_LOCK(pmap);
LOCK_TABLE();
if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
- pt = moea64_pvo_to_pte(pvo, -1);
+ pt = moea64_pvo_to_pte(pvo);
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
if (pt != NULL) {
@@ -1573,7 +1857,9 @@ moea64_remove_write(mmu_t mmu, vm_page_t m)
lo |= pvo->pvo_pte.lpte.pte_lo;
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
moea64_pte_change(pt, &pvo->pvo_pte.lpte,
- pvo->pvo_pmap, PVO_VADDR(pvo));
+ pvo->pvo_vpn);
+ if (pvo->pvo_pmap == kernel_pmap)
+ isync();
}
}
UNLOCK_TABLE();
@@ -1620,7 +1906,8 @@ moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
#if 0
if (!pmap_bootstrapped) {
if (va >= VM_MIN_KERNEL_ADDRESS && va < virtual_end)
- panic("Trying to enter an address in KVA -- %#x!\n",pa);
+ panic("Trying to enter an address in KVA -- %#"
+ PRIxPTR "!\n",pa);
}
#endif
@@ -1632,7 +1919,7 @@ moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
PVO_WIRED | VM_PROT_EXECUTE);
if (error != 0 && error != ENOENT)
- panic("moea64_kenter: failed to enter va %#x pa %#x: %d", va,
+ panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
pa, error);
/*
@@ -1662,9 +1949,10 @@ moea64_kextract(mmu_t mmu, vm_offset_t va)
return (va);
PMAP_LOCK(kernel_pmap);
- pvo = moea64_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL);
- KASSERT(pvo != NULL, ("moea64_kextract: no addr found"));
- pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+ pvo = moea64_pvo_find_va(kernel_pmap, va);
+ KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR,
+ va));
+ pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) + (va - PVO_VADDR(pvo));
PMAP_UNLOCK(kernel_pmap);
return (pa);
}
@@ -1754,29 +2042,20 @@ moea64_page_wired_mappings(mmu_t mmu, vm_page_t m)
return (count);
}
-static u_int moea64_vsidcontext;
+static uintptr_t moea64_vsidcontext;
-void
-moea64_pinit(mmu_t mmu, pmap_t pmap)
-{
- int i, mask;
- u_int entropy;
-
- PMAP_LOCK_INIT(pmap);
+uintptr_t
+moea64_get_unique_vsid(void) {
+ u_int entropy;
+ register_t hash;
+ uint32_t mask;
+ int i;
entropy = 0;
__asm __volatile("mftb %0" : "=r"(entropy));
- if (pmap_bootstrapped)
- pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, (vm_offset_t)pmap);
- else
- pmap->pmap_phys = pmap;
-
- /*
- * Allocate some segment registers for this pmap.
- */
- for (i = 0; i < NPMAPS; i += VSID_NBPW) {
- u_int hash, n;
+ for (i = 0; i < NVSIDS; i += VSID_NBPW) {
+ u_int n;
/*
* Create a new value by mutiplying by a prime and adding in
@@ -1786,12 +2065,12 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
* instead of a multiply.)
*/
moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy;
- hash = moea64_vsidcontext & (NPMAPS - 1);
+ hash = moea64_vsidcontext & (NVSIDS - 1);
if (hash == 0) /* 0 is special, avoid it */
continue;
n = hash >> 5;
mask = 1 << (hash & (VSID_NBPW - 1));
- hash = (moea64_vsidcontext & 0xfffff);
+ hash = (moea64_vsidcontext & VSID_HASHMASK);
if (moea64_vsid_bitmap[n] & mask) { /* collision? */
/* anything free in this bucket? */
if (moea64_vsid_bitmap[n] == 0xffffffff) {
@@ -1800,18 +2079,49 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
}
i = ffs(~moea64_vsid_bitmap[i]) - 1;
mask = 1 << i;
- hash &= 0xfffff & ~(VSID_NBPW - 1);
+ hash &= VSID_HASHMASK & ~(VSID_NBPW - 1);
hash |= i;
}
moea64_vsid_bitmap[n] |= mask;
- for (i = 0; i < 16; i++) {
- pmap->pm_sr[i] = VSID_MAKE(i, hash);
- }
- return;
+ return (hash);
}
- panic("moea64_pinit: out of segments");
+ panic("%s: out of segments",__func__);
+}
+
+#ifdef __powerpc64__
+void
+moea64_pinit(mmu_t mmu, pmap_t pmap)
+{
+ PMAP_LOCK_INIT(pmap);
+
+ SPLAY_INIT(&pmap->pm_slbtree);
+ pmap->pm_slb = slb_alloc_user_cache();
+}
+#else
+void
+moea64_pinit(mmu_t mmu, pmap_t pmap)
+{
+ int i;
+ register_t hash;
+
+ PMAP_LOCK_INIT(pmap);
+
+ if (pmap_bootstrapped)
+ pmap->pmap_phys = (pmap_t)moea64_kextract(mmu,
+ (vm_offset_t)pmap);
+ else
+ pmap->pmap_phys = pmap;
+
+ /*
+ * Allocate some segment registers for this pmap.
+ */
+ hash = moea64_get_unique_vsid();
+
+ for (i = 0; i < 16; i++)
+ pmap->pm_sr[i] = VSID_MAKE(i, hash);
}
+#endif
/*
* Initialize the pmap associated with process 0.
@@ -1832,7 +2142,6 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
{
struct pvo_entry *pvo;
struct lpte *pt;
- int pteidx;
CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva,
eva, prot);
@@ -1849,7 +2158,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
vm_page_lock_queues();
PMAP_LOCK(pm);
for (; sva < eva; sva += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(pm, sva, &pteidx);
+ pvo = moea64_pvo_find_va(pm, sva);
if (pvo == NULL)
continue;
@@ -1858,7 +2167,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
* copy.
*/
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, pteidx);
+ pt = moea64_pvo_to_pte(pvo);
/*
* Change the protection of the page.
@@ -1873,8 +2182,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
* If the PVO is in the page table, update that pte as well.
*/
if (pt != NULL) {
- moea64_pte_change(pt, &pvo->pvo_pte.lpte,
- pvo->pvo_pmap, PVO_VADDR(pvo));
+ moea64_pte_change(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
if ((pvo->pvo_pte.lpte.pte_lo &
(LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
moea64_syncicache(pm, sva,
@@ -1917,20 +2225,34 @@ moea64_qremove(mmu_t mmu, vm_offset_t va, int count)
}
void
-moea64_release(mmu_t mmu, pmap_t pmap)
+moea64_release_vsid(uint64_t vsid)
{
int idx, mask;
+
+ idx = vsid & (NVSIDS-1);
+ mask = 1 << (idx % VSID_NBPW);
+ idx /= VSID_NBPW;
+ moea64_vsid_bitmap[idx] &= ~mask;
+}
+
+
+void
+moea64_release(mmu_t mmu, pmap_t pmap)
+{
/*
- * Free segment register's VSID
+ * Free segment registers' VSIDs
*/
+ #ifdef __powerpc64__
+ free_vsids(pmap);
+ slb_free_user_cache(pmap->pm_slb);
+ #else
if (pmap->pm_sr[0] == 0)
panic("moea64_release");
- idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1);
- mask = 1 << (idx % VSID_NBPW);
- idx /= VSID_NBPW;
- moea64_vsid_bitmap[idx] &= ~mask;
+ moea64_release_vsid(pmap->pm_sr[0]);
+ #endif
+
PMAP_LOCK_DESTROY(pmap);
}
@@ -1941,15 +2263,13 @@ void
moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{
struct pvo_entry *pvo;
- int pteidx;
vm_page_lock_queues();
PMAP_LOCK(pm);
for (; sva < eva; sva += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(pm, sva, &pteidx);
- if (pvo != NULL) {
- moea64_pvo_remove(pvo, pteidx);
- }
+ pvo = moea64_pvo_find_va(pm, sva);
+ if (pvo != NULL)
+ moea64_pvo_remove(pvo);
}
vm_page_unlock_queues();
PMAP_UNLOCK(pm);
@@ -1974,7 +2294,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m)
MOEA_PVO_CHECK(pvo); /* sanity check */
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
- moea64_pvo_remove(pvo, -1);
+ moea64_pvo_remove(pvo);
PMAP_UNLOCK(pmap);
}
if ((m->flags & PG_WRITEABLE) && moea64_is_modified(mmu, m)) {
@@ -2032,25 +2352,45 @@ static void
tlbia(void)
{
vm_offset_t i;
+ #ifndef __powerpc64__
register_t msr, scratch;
+ #endif
+
+ TLBSYNC();
for (i = 0; i < 0xFF000; i += 0x00001000) {
+ #ifdef __powerpc64__
+ __asm __volatile("tlbiel %0" :: "r"(i));
+ #else
__asm __volatile("\
mfmsr %0; \
mr %1, %0; \
insrdi %1,%3,1,0; \
mtmsrd %1; \
- ptesync; \
+ isync; \
\
tlbiel %2; \
\
mtmsrd %0; \
- eieio; \
- tlbsync; \
- ptesync;"
+ isync;"
: "=r"(msr), "=r"(scratch) : "r"(i), "r"(1));
+ #endif
}
+
+ EIEIO();
+ TLBSYNC();
+}
+
+#ifdef __powerpc64__
+static void
+slbia(void)
+{
+ register_t seg0;
+
+ __asm __volatile ("slbia");
+ __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
}
+#endif
static int
moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
@@ -2084,7 +2424,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
*/
va &= ~ADDR_POFF;
vsid = va_to_vsid(pm, va);
- ptegidx = va_to_pteg(vsid, va);
+ ptegidx = va_to_pteg(vsid, va, flags & PVO_LARGE);
/*
* Remove any existing mapping for this page. Reuse the pvo entry if
@@ -2097,10 +2437,18 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa &&
(pvo->pvo_pte.lpte.pte_lo & LPTE_PP) ==
(pte_lo & LPTE_PP)) {
+ if (!(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) {
+ /* Re-insert if spilled */
+ i = moea64_pte_insert(ptegidx,
+ &pvo->pvo_pte.lpte);
+ if (i >= 0)
+ PVO_PTEGIDX_SET(pvo, i);
+ moea64_pte_overflow--;
+ }
UNLOCK_TABLE();
return (0);
}
- moea64_pvo_remove(pvo, -1);
+ moea64_pvo_remove(pvo);
break;
}
}
@@ -2110,7 +2458,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
*/
if (bootstrap) {
if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) {
- panic("moea64_enter: bpvo pool exhausted, %d, %d, %d",
+ panic("moea64_enter: bpvo pool exhausted, %d, %d, %zd",
moea64_bpvo_pool_index, BPVO_POOL_SIZE,
BPVO_POOL_SIZE * sizeof(struct pvo_entry));
}
@@ -2136,6 +2484,8 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
moea64_pvo_entries++;
pvo->pvo_vaddr = va;
+ pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT)
+ | (vsid << 16);
pvo->pvo_pmap = pm;
LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink);
pvo->pvo_vaddr &= ~ADDR_POFF;
@@ -2150,9 +2500,11 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
pvo->pvo_vaddr |= PVO_BOOTSTRAP;
if (flags & PVO_FAKE)
pvo->pvo_vaddr |= PVO_FAKE;
+ if (flags & PVO_LARGE)
+ pvo->pvo_vaddr |= PVO_LARGE;
moea64_pte_create(&pvo->pvo_pte.lpte, vsid, va,
- (uint64_t)(pa) | pte_lo);
+ (uint64_t)(pa) | pte_lo, flags);
/*
* Remember if the list was empty and therefore will be the first
@@ -2162,8 +2514,10 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
first = 1;
LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
- if (pvo->pvo_vaddr & PVO_WIRED)
+ if (pvo->pvo_vaddr & PVO_WIRED) {
+ pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED;
pm->pm_stats.wired_count++;
+ }
pm->pm_stats.resident_count++;
/*
@@ -2182,11 +2536,20 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
UNLOCK_TABLE();
+#ifdef __powerpc64__
+ /*
+ * Make sure all our bootstrap mappings are in the SLB as soon
+ * as virtual memory is switched on.
+ */
+ if (!pmap_bootstrapped)
+ moea64_bootstrap_slb_prefault(va, flags & PVO_LARGE);
+#endif
+
return (first ? ENOENT : 0);
}
static void
-moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
+moea64_pvo_remove(struct pvo_entry *pvo)
{
struct lpte *pt;
@@ -2195,10 +2558,9 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
* save the ref & cfg bits).
*/
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, pteidx);
+ pt = moea64_pvo_to_pte(pvo);
if (pt != NULL) {
- moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap,
- PVO_VADDR(pvo));
+ moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
PVO_PTEGIDX_CLR(pvo);
} else {
moea64_pte_overflow--;
@@ -2244,39 +2606,35 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
moea64_pvo_remove_calls++;
}
-static __inline int
-moea64_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
-{
-
- /*
- * We can find the actual pte entry without searching by grabbing
- * the PTEG index from 3 unused bits in pvo_vaddr and by
- * noticing the HID bit.
- */
- if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
- ptegidx ^= moea64_pteg_mask;
-
- return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo));
-}
-
static struct pvo_entry *
-moea64_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
+moea64_pvo_find_va(pmap_t pm, vm_offset_t va)
{
struct pvo_entry *pvo;
int ptegidx;
uint64_t vsid;
+ #ifdef __powerpc64__
+ struct slb slb;
+ /* The page is not mapped if the segment isn't */
+ if (va_to_slb_entry(pm, va, &slb) != 0)
+ return NULL;
+
+ vsid = (slb.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT;
+ if (slb.slbv & SLBV_L)
+ va &= ~moea64_large_page_mask;
+ else
+ va &= ~ADDR_POFF;
+ ptegidx = va_to_pteg(vsid, va, slb.slbv & SLBV_L);
+ #else
va &= ~ADDR_POFF;
vsid = va_to_vsid(pm, va);
- ptegidx = va_to_pteg(vsid, va);
+ ptegidx = va_to_pteg(vsid, va, 0);
+ #endif
LOCK_TABLE();
LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
- if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
- if (pteidx_p)
- *pteidx_p = moea64_pvo_pte_index(pvo, ptegidx);
+ if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va)
break;
- }
}
UNLOCK_TABLE();
@@ -2284,23 +2642,34 @@ moea64_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
}
static struct lpte *
-moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
+moea64_pvo_to_pte(const struct pvo_entry *pvo)
{
- struct lpte *pt;
+ struct lpte *pt;
+ int pteidx, ptegidx;
+ uint64_t vsid;
+
+ ASSERT_TABLE_LOCK();
+
+ /* If the PTEG index is not set, then there is no page table entry */
+ if (!PVO_PTEGIDX_ISSET(pvo))
+ return (NULL);
/*
- * If we haven't been supplied the ptegidx, calculate it.
+ * Calculate the ptegidx
*/
- if (pteidx == -1) {
- int ptegidx;
- uint64_t vsid;
+ vsid = PVO_VSID(pvo);
+ ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
+ pvo->pvo_vaddr & PVO_LARGE);
- vsid = va_to_vsid(pvo->pvo_pmap, PVO_VADDR(pvo));
- ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo));
- pteidx = moea64_pvo_pte_index(pvo, ptegidx);
- }
+ /*
+ * We can find the actual pte entry without searching by grabbing
+ * the PTEG index from 3 unused bits in pvo_vaddr and by
+ * noticing the HID bit.
+ */
+ if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
+ ptegidx ^= moea64_pteg_mask;
- pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
+ pteidx = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo);
if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
!PVO_PTEGIDX_ISSET(pvo)) {
@@ -2314,6 +2683,7 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
"pvo but no valid pte", pvo);
}
+ pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) ==
LPTE_VALID) {
if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) {
@@ -2329,7 +2699,6 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
(uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo));
}
- ASSERT_TABLE_LOCK();
return (pt);
}
@@ -2341,10 +2710,37 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
return (NULL);
}
+static __inline int
+moea64_pte_spillable_ident(u_int ptegidx)
+{
+ struct lpte *pt;
+ int i, j, k;
+
+ /* Start at a random slot */
+ i = mftb() % 8;
+ k = -1;
+ for (j = 0; j < 8; j++) {
+ pt = &moea64_pteg_table[ptegidx].pt[(i + j) % 8];
+ if (pt->pte_hi & (LPTE_LOCKED | LPTE_WIRED))
+ continue;
+
+ /* This is a candidate, so remember it */
+ k = (i + j) % 8;
+
+ /* Try to get a page that has not been used lately */
+ if (!(pt->pte_lo & LPTE_REF))
+ return (k);
+ }
+
+ return (k);
+}
+
static int
moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
{
struct lpte *pt;
+ struct pvo_entry *pvo;
+ u_int pteg_bktidx;
int i;
ASSERT_TABLE_LOCK();
@@ -2352,9 +2748,9 @@ moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
/*
* First try primary hash.
*/
- for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
- if ((pt->pte_hi & LPTE_VALID) == 0 &&
- (pt->pte_hi & LPTE_LOCKED) == 0) {
+ pteg_bktidx = ptegidx;
+ for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
+ if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
pvo_pt->pte_hi &= ~LPTE_HID;
moea64_pte_set(pt, pvo_pt);
return (i);
@@ -2364,19 +2760,66 @@ moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
/*
* Now try secondary hash.
*/
- ptegidx ^= moea64_pteg_mask;
-
- for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
- if ((pt->pte_hi & LPTE_VALID) == 0 &&
- (pt->pte_hi & LPTE_LOCKED) == 0) {
+ pteg_bktidx ^= moea64_pteg_mask;
+ for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
+ if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
pvo_pt->pte_hi |= LPTE_HID;
moea64_pte_set(pt, pvo_pt);
return (i);
}
}
- panic("moea64_pte_insert: overflow");
- return (-1);
+ /*
+ * Out of luck. Find a PTE to sacrifice.
+ */
+ pteg_bktidx = ptegidx;
+ i = moea64_pte_spillable_ident(pteg_bktidx);
+ if (i < 0) {
+ pteg_bktidx ^= moea64_pteg_mask;
+ i = moea64_pte_spillable_ident(pteg_bktidx);
+ }
+
+ if (i < 0) {
+ /* No freeable slots in either PTEG? We're hosed. */
+ panic("moea64_pte_insert: overflow");
+ return (-1);
+ }
+
+ if (pteg_bktidx == ptegidx)
+ pvo_pt->pte_hi &= ~LPTE_HID;
+ else
+ pvo_pt->pte_hi |= LPTE_HID;
+
+ /*
+ * Synchronize the sacrifice PTE with its PVO, then mark both
+ * invalid. The PVO will be reused when/if the VM system comes
+ * here after a fault.
+ */
+ pt = &moea64_pteg_table[pteg_bktidx].pt[i];
+
+ if (pt->pte_hi & LPTE_HID)
+ pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
+
+ LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) {
+ if (pvo->pvo_pte.lpte.pte_hi == pt->pte_hi) {
+ KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID,
+ ("Invalid PVO for valid PTE!"));
+ moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
+ PVO_PTEGIDX_CLR(pvo);
+ moea64_pte_overflow++;
+ break;
+ }
+ }
+
+ KASSERT(pvo->pvo_pte.lpte.pte_hi == pt->pte_hi,
+ ("Unable to find PVO for spilled PTE"));
+
+ /*
+ * Set the new PTE.
+ */
+ moea64_pte_set(pt, pvo_pt);
+
+ return (i);
}
static boolean_t
@@ -2420,7 +2863,7 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit)
* ptebit is set, cache it and return success.
*/
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, -1);
+ pt = moea64_pvo_to_pte(pvo);
if (pt != NULL) {
moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
@@ -2471,12 +2914,12 @@ moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
MOEA_PVO_CHECK(pvo); /* sanity check */
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, -1);
+ pt = moea64_pvo_to_pte(pvo);
if (pt != NULL) {
moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
count++;
- moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit);
+ moea64_pte_clear(pt, pvo->pvo_vpn, ptebit);
}
}
pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
@@ -2497,7 +2940,7 @@ moea64_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size)
PMAP_LOCK(kernel_pmap);
for (ppa = pa & ~ADDR_POFF; ppa < pa + size; ppa += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(kernel_pmap, ppa, NULL);
+ pvo = moea64_pvo_find_va(kernel_pmap, ppa);
if (pvo == NULL ||
(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) != ppa) {
error = EFAULT;
@@ -2563,7 +3006,7 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz)
while (sz > 0) {
lim = round_page(va);
len = MIN(lim - va, sz);
- pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
if (pvo != NULL) {
pa = (pvo->pvo_pte.pte.pte_lo & LPTE_RPGN) |
(va & ADDR_POFF);
OpenPOWER on IntegriCloud