summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/amd64/amd64/genassym.c12
-rw-r--r--sys/amd64/amd64/locore.S9
-rw-r--r--sys/amd64/amd64/machdep.c86
-rw-r--r--sys/amd64/amd64/mem.c3
-rw-r--r--sys/amd64/amd64/nexus.c4
-rw-r--r--sys/amd64/amd64/pmap.c1003
-rw-r--r--sys/amd64/amd64/trap.c2
-rw-r--r--sys/amd64/amd64/vm_machdep.c2
-rw-r--r--sys/amd64/include/bus_amd64.h12
-rw-r--r--sys/amd64/include/param.h12
-rw-r--r--sys/amd64/include/pmap.h79
-rw-r--r--sys/amd64/include/vmparam.h36
-rw-r--r--sys/conf/kern.mk2
13 files changed, 656 insertions, 606 deletions
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 3ccad23..7f84e3d 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -99,8 +99,11 @@ ASSYM(KSTACK_PAGES, KSTACK_PAGES);
ASSYM(PAGE_SIZE, PAGE_SIZE);
ASSYM(NPTEPG, NPTEPG);
ASSYM(NPDEPG, NPDEPG);
-ASSYM(NPDEPTD, NPDEPTD);
-ASSYM(NPGPTD, NPGPTD);
+ASSYM(addr_PTmap, addr_PTmap);
+ASSYM(addr_PDmap, addr_PDmap);
+ASSYM(addr_PDPmap, addr_PDPmap);
+ASSYM(addr_PML4map, addr_PML4map);
+ASSYM(addr_PML4pml4e, addr_PML4pml4e);
ASSYM(PDESIZE, sizeof(pd_entry_t));
ASSYM(PTESIZE, sizeof(pt_entry_t));
ASSYM(PTESHIFT, PTESHIFT);
@@ -109,9 +112,14 @@ ASSYM(PAGE_MASK, PAGE_MASK);
ASSYM(PDRSHIFT, PDRSHIFT);
ASSYM(PDPSHIFT, PDPSHIFT);
ASSYM(PML4SHIFT, PML4SHIFT);
+ASSYM(val_KPDPI, KPDPI);
+ASSYM(val_KPML4I, KPML4I);
+ASSYM(val_PML4PML4I, PML4PML4I);
ASSYM(USRSTACK, USRSTACK);
ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
ASSYM(KERNBASE, KERNBASE);
+ASSYM(DMAP_MIN_ADDRESS, DMAP_MIN_ADDRESS);
+ASSYM(DMAP_MAX_ADDRESS, DMAP_MAX_ADDRESS);
ASSYM(MCLBYTES, MCLBYTES);
ASSYM(PCB_CR3, offsetof(struct pcb, pcb_cr3));
ASSYM(PCB_R15, offsetof(struct pcb, pcb_r15));
diff --git a/sys/amd64/amd64/locore.S b/sys/amd64/amd64/locore.S
index 04d2ac5..0d2c2cc 100644
--- a/sys/amd64/amd64/locore.S
+++ b/sys/amd64/amd64/locore.S
@@ -36,8 +36,15 @@
/*
* Compiled KERNBASE location
*/
- .globl kernbase
+ .globl kernbase,loc_PTmap,loc_PDmap,loc_PDPmap,loc_PML4map,loc_PML4pml4e,dmapbase,dmapend
.set kernbase,KERNBASE
+ .set loc_PTmap,addr_PTmap
+ .set loc_PDmap,addr_PDmap
+ .set loc_PDPmap,addr_PDPmap
+ .set loc_PML4map,addr_PML4map
+ .set loc_PML4pml4e,addr_PML4pml4e
+ .set dmapbase,DMAP_MIN_ADDRESS
+ .set dmapend,DMAP_MAX_ADDRESS
.text
/**********************************************************************
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index c9c4d93..6afe51a 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -133,11 +133,6 @@ u_long atdevbase;
u_int64_t modulep; /* phys addr of metadata table */
u_int64_t physfree; /* first free page after kernel */
-u_int64_t IdlePTD; /* phys addr of kernel PTD */
-u_int64_t IdlePDP; /* phys addr of kernel level 3 */
-u_int64_t IdlePML4; /* phys addr of kernel level 4 */
-struct user *proc0uarea; /* address of proc 0 uarea space */
-vm_offset_t proc0kstack; /* address of proc 0 kstack space */
int cold = 1;
@@ -945,7 +940,7 @@ physmap_done:
physmap[physmap_idx + 1] = ptoa((vm_paddr_t)Maxmem);
/* call pmap initialization to make new kernel address space */
- pmap_bootstrap(first, 0);
+ pmap_bootstrap(&first);
/*
* Size up each available chunk of physical memory.
@@ -1086,69 +1081,6 @@ allocpages(int n)
return (ret);
}
-static void
-create_pagetables(void)
-{
- u_int64_t p0kpa;
- u_int64_t p0upa;
- u_int64_t KPTphys;
- int i;
-
- /* Allocate pages */
- KPTphys = allocpages(NKPT);
- IdlePML4 = allocpages(NKPML4E);
- IdlePDP = allocpages(NKPDPE);
- IdlePTD = allocpages(NPGPTD);
- p0upa = allocpages(UAREA_PAGES);
- p0kpa = allocpages(KSTACK_PAGES);
-
- proc0uarea = (struct user *)(p0upa + KERNBASE);
- proc0kstack = p0kpa + KERNBASE;
-
- /* Fill in the underlying page table pages */
- /* Read-only from zero to physfree */
- /* XXX not fully used, underneath 2M pages */
- for (i = 0; (i << PAGE_SHIFT) < physfree; i++) {
- ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
- ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V;
- }
-
- /* Now map the page tables at their location within PTmap */
- for (i = 0; i < NKPT; i++) {
- ((pd_entry_t *)IdlePTD)[i + KPTDI] = KPTphys + (i << PAGE_SHIFT);
- ((pd_entry_t *)IdlePTD)[i + KPTDI] |= PG_RW | PG_V;
- }
-
- /* Map from zero to end of allocations under 2M pages */
- /* This replaces some of the PTD entries above */
- for (i = 0; (i << PDRSHIFT) < physfree; i++) {
- ((pd_entry_t *)IdlePTD)[i] = i << PDRSHIFT;
- ((pd_entry_t *)IdlePTD)[i] |= PG_RW | PG_V | PG_PS;
- }
-
- /* Now map the page tables at their location within PTmap */
- for (i = 0; i < NKPT; i++) {
- ((pd_entry_t *)IdlePTD)[i] = KPTphys + (i << PAGE_SHIFT);
- ((pd_entry_t *)IdlePTD)[i] |= PG_RW | PG_V;
- }
-
- /* Now map the PTD at the top of the PTmap (ie: PTD[]) */
- for (i = 0; i < NPGPTD; i++) {
- ((pd_entry_t *)IdlePTD)[i + PTDPTDI] = IdlePTD + (i << PAGE_SHIFT);
- ((pd_entry_t *)IdlePTD)[i + PTDPTDI] |= PG_RW | PG_V;
- }
-
- /* And connect up the PTD to the PDP */
- for (i = 0; i < NPGPTD; i++) {
- ((pdp_entry_t *)IdlePDP)[i] = IdlePTD + (i << PAGE_SHIFT);
- ((pdp_entry_t *)IdlePDP)[i] |= PG_RW | PG_V | PG_U;
- }
-
- /* And connect up the PDP to the PML4 */
- ((pdp_entry_t *)IdlePML4)[0] = IdlePDP;
- ((pdp_entry_t *)IdlePML4)[0] |= PG_RW | PG_V | PG_U;
-}
-
void
hammer_time(void)
{
@@ -1157,18 +1089,14 @@ hammer_time(void)
struct region_descriptor r_gdt, r_idt;
struct pcpu *pc;
u_int64_t msr;
+ char *env;
/* Turn on PTE NX (no execute) bit */
msr = rdmsr(MSR_EFER) | EFER_NXE;
wrmsr(MSR_EFER, msr);
- create_pagetables();
- /* XXX do %cr0 as well */
- load_cr4(rcr4() | CR4_PGE | CR4_PSE);
- load_cr3(IdlePML4);
-
- proc0.p_uarea = proc0uarea;
- thread0.td_kstack = proc0kstack;
+ proc0.p_uarea = (struct user *)(allocpages(UAREA_PAGES) + KERNBASE);
+ thread0.td_kstack = allocpages(KSTACK_PAGES) + KERNBASE;
thread0.td_pcb = (struct pcb *)
(thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1;
atdevbase = ISA_HOLE_START + KERNBASE;
@@ -1310,8 +1238,12 @@ hammer_time(void)
/* setup proc 0's pcb */
thread0.td_pcb->pcb_flags = 0; /* XXXKSE */
- thread0.td_pcb->pcb_cr3 = IdlePML4;
+ thread0.td_pcb->pcb_cr3 = KPML4phys;
thread0.td_frame = &proc0_tf;
+
+ env = getenv("kernelname");
+ if (env != NULL)
+ strlcpy(kernelname, env, sizeof(kernelname));
}
void
diff --git a/sys/amd64/amd64/mem.c b/sys/amd64/amd64/mem.c
index 38a6e13..dd403f7 100644
--- a/sys/amd64/amd64/mem.c
+++ b/sys/amd64/amd64/mem.c
@@ -63,6 +63,7 @@
#include <machine/frame.h>
#include <machine/psl.h>
#include <machine/specialreg.h>
+#include <machine/vmparam.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -177,7 +178,7 @@ mmrw(dev_t dev, struct uio *uio, int flags)
addr = trunc_page(uio->uio_offset);
eaddr = round_page(uio->uio_offset + c);
- if (addr < (vm_offset_t)VADDR(0, 0, PTDPTDI, 0))
+ if (addr < (vm_offset_t)KERNBASE)
return (EFAULT);
for (; addr < eaddr; addr += PAGE_SIZE)
if (pmap_extract(kernel_pmap, addr) == 0)
diff --git a/sys/amd64/amd64/nexus.c b/sys/amd64/amd64/nexus.c
index a3cc266..a854252 100644
--- a/sys/amd64/amd64/nexus.c
+++ b/sys/amd64/amd64/nexus.c
@@ -355,8 +355,8 @@ nexus_activate_resource(device_t bus, device_t child, int type, int rid,
*/
vaddr = (caddr_t)(uintptr_t)(KERNBASE + rman_get_start(r));
} else {
- u_int32_t paddr;
- u_int32_t psize;
+ u_int64_t paddr;
+ u_int64_t psize;
u_int32_t poffs;
paddr = rman_get_start(r);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 1700077d..d3cdb59 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -125,6 +125,7 @@
#include <vm/vm_pageout.h>
#include <vm/vm_pager.h>
#include <vm/uma.h>
+#include <vm/uma_int.h>
#include <machine/cpu.h>
#include <machine/cputypes.h>
@@ -149,41 +150,35 @@
#endif
/*
- * Get PDEs and PTEs for user/kernel address space
- */
-#define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
-#define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
-
-#define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0)
-#define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0)
-#define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0)
-#define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0)
-#define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0)
-
-#define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W))
-#define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
-
-/*
* Given a map and a machine independent protection code,
* convert to a vax protection code.
*/
#define pte_prot(m, p) (protection_codes[p])
-static int protection_codes[8];
+static pt_entry_t protection_codes[8];
struct pmap kernel_pmap_store;
LIST_HEAD(pmaplist, pmap);
static struct pmaplist allpmaps;
static struct mtx allpmaps_lock;
-vm_paddr_t avail_start; /* PA of first available physical page */
-vm_paddr_t avail_end; /* PA of last available physical page */
+vm_paddr_t avail_start; /* PA of first available physical page */
+vm_paddr_t avail_end; /* PA of last available physical page */
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */
static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */
static int nkpt;
+static int ndmpdp;
vm_offset_t kernel_vm_end;
+static u_int64_t KPTphys; /* phys addr of kernel level 1 */
+static u_int64_t KPDphys; /* phys addr of kernel level 2 */
+static u_int64_t KPDPphys; /* phys addr of kernel level 3 */
+u_int64_t KPML4phys; /* phys addr of kernel level 4 */
+
+static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */
+static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */
+
/*
* Data for the pv entry allocation mechanism
*/
@@ -196,10 +191,8 @@ int pmap_pagedaemon_waken;
* All those kernel PT submaps that BSD is so fond of
*/
pt_entry_t *CMAP1 = 0;
-static pt_entry_t *CMAP2, *CMAP3, *ptmmap;
+static pt_entry_t *ptmmap;
caddr_t CADDR1 = 0, ptvmmap = 0;
-static caddr_t CADDR2, CADDR3;
-static struct mtx CMAPCADDR12_lock;
static pt_entry_t *msgbufmap;
struct msgbuf *msgbufp = 0;
@@ -209,12 +202,9 @@ struct msgbuf *msgbufp = 0;
static pt_entry_t *pt_crashdumpmap;
static caddr_t crashdumpmap;
-static pt_entry_t *PMAP1 = 0;
-static pt_entry_t *PADDR1 = 0;
-
static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
static pv_entry_t get_pv_entry(void);
-static void i386_protection_init(void);
+static void amd64_protection_init(void);
static __inline void pmap_changebit(vm_page_t m, int bit, boolean_t setem);
static vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va,
@@ -228,7 +218,7 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va,
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va);
-static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex);
+static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex);
static vm_page_t pmap_page_lookup(vm_object_t object, vm_pindex_t pindex);
static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t);
static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
@@ -252,10 +242,195 @@ pmap_kmem_choose(vm_offset_t addr)
return newaddr;
}
+/********************/
+/* Inline functions */
+/********************/
+
+/* Return a non-clipped PD index for a given VA */
+static __inline vm_pindex_t
+pmap_pde_pindex(vm_offset_t va)
+{
+ return va >> PDRSHIFT;
+}
+
+
+/* Return various clipped indexes for a given VA */
+static __inline vm_pindex_t
+pmap_pte_index(vm_offset_t va)
+{
+
+ return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pde_index(vm_offset_t va)
+{
+
+ return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pdpe_index(vm_offset_t va)
+{
+
+ return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
+}
+
+static __inline vm_pindex_t
+pmap_pml4e_index(vm_offset_t va)
+{
+
+ return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
+}
+
+/* Return a pointer to the PML4 slot that corresponds to a VA */
+static __inline pml4_entry_t *
+pmap_pml4e(pmap_t pmap, vm_offset_t va)
+{
+
+ if (!pmap)
+ return NULL;
+ return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
+}
+
+/* Return a pointer to the PDP slot that corresponds to a VA */
+static __inline pdp_entry_t *
+pmap_pdpe(pmap_t pmap, vm_offset_t va)
+{
+ pml4_entry_t *pml4e;
+ pdp_entry_t *pdpe;
+
+ pml4e = pmap_pml4e(pmap, va);
+ if (pml4e == NULL || (*pml4e & PG_V) == 0)
+ return NULL;
+ pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
+ return (&pdpe[pmap_pdpe_index(va)]);
+}
+
+/* Return a pointer to the PD slot that corresponds to a VA */
+static __inline pd_entry_t *
+pmap_pde(pmap_t pmap, vm_offset_t va)
+{
+ pdp_entry_t *pdpe;
+ pd_entry_t *pde;
+
+ pdpe = pmap_pdpe(pmap, va);
+ if (pdpe == NULL || (*pdpe & PG_V) == 0)
+ return NULL;
+ pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
+ return (&pde[pmap_pde_index(va)]);
+}
+
+/* Return a pointer to the PT slot that corresponds to a VA */
+static __inline pt_entry_t *
+pmap_pte(pmap_t pmap, vm_offset_t va)
+{
+ pd_entry_t *pde;
+ pt_entry_t *pte;
+
+ pde = pmap_pde(pmap, va);
+ if (pde == NULL || (*pde & PG_V) == 0)
+ return NULL;
+ pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
+ return (&pte[pmap_pte_index(va)]);
+}
+
+
+PMAP_INLINE pt_entry_t *
+vtopte(vm_offset_t va)
+{
+ u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
+
+ return (PTmap + (amd64_btop(va) & mask));
+}
+
+static u_int64_t
+allocpages(int n)
+{
+ u_int64_t ret;
+
+ ret = avail_start;
+ bzero((void *)ret, n * PAGE_SIZE);
+ avail_start += n * PAGE_SIZE;
+ return (ret);
+}
+
+static void
+create_pagetables(void)
+{
+ int i;
+
+ /* Allocate pages */
+ KPTphys = allocpages(NKPT);
+ KPML4phys = allocpages(1);
+ KPDPphys = allocpages(NKPML4E);
+ KPDphys = allocpages(NKPDPE);
+
+ ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
+ if (ndmpdp < 1)
+ ndmpdp = 1;
+ DMPDPphys = allocpages(NDMPML4E);
+ DMPDphys = allocpages(ndmpdp);
+
+ /* Fill in the underlying page table pages */
+ /* Read-only from zero to physfree */
+ /* XXX not fully used, underneath 2M pages */
+ for (i = 0; (i << PAGE_SHIFT) < avail_start; i++) {
+ ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
+ ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V;
+ }
+
+ /* Now map the page tables at their location within PTmap */
+ for (i = 0; i < NKPT; i++) {
+ ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
+ ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
+ }
+
+#if 0
+ /* Map from zero to end of allocations under 2M pages */
+ /* This replaces some of the KPTphys entries above */
+ for (i = 0; (i << PDRSHIFT) < avail_start; i++) {
+ ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
+ ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS;
+ }
+#endif
+
+ /* And connect up the PD to the PDP */
+ for (i = 0; i < NKPDPE; i++) {
+ ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys + (i << PAGE_SHIFT);
+ ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
+ }
+
+
+ /* Now set up the direct map space using 2MB pages */
+ for (i = 0; i < NPDEPG * ndmpdp; i++) {
+ ((pd_entry_t *)DMPDphys)[i] = i << PDRSHIFT;
+ ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS;
+ }
+
+ /* And the direct map space's PDP */
+ for (i = 0; i < ndmpdp; i++) {
+ ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (i << PAGE_SHIFT);
+ ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
+ }
+
+ /* And recursively map PML4 to itself in order to get PTmap */
+ ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
+ ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
+
+ /* Connect the Direct Map slot up to the PML4 */
+ ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
+ ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
+
+ /* Connect the KVA slot up to the PML4 */
+ ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
+ ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
+}
+
/*
* Bootstrap the system enough to run with virtual memory.
*
- * On the i386 this is called after mapping has already been enabled
+ * On amd64 this is called after mapping has already been enabled
* and just syncs the pmap module with what has already been done.
* [We can't call it easily with mapping off since the kernel is not
* mapped with PA == VA, hence we would have to relocate every address
@@ -263,40 +438,39 @@ pmap_kmem_choose(vm_offset_t addr)
* (physical) address starting relative to 0]
*/
void
-pmap_bootstrap(firstaddr, loadaddr)
- vm_paddr_t firstaddr;
- vm_paddr_t loadaddr;
+pmap_bootstrap(firstaddr)
+ vm_paddr_t *firstaddr;
{
vm_offset_t va;
pt_entry_t *pte;
- int i;
- avail_start = firstaddr;
+ avail_start = *firstaddr;
/*
- * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too
- * large. It should instead be correctly calculated in locore.s and
- * not based on 'first' (which is a physical address, not a virtual
- * address, for the start of unused physical memory). The kernel
- * page tables are NOT double mapped and thus should not be included
- * in this calculation.
+ * Create an initial set of page tables to run the kernel in.
*/
- virtual_avail = (vm_offset_t) KERNBASE + firstaddr;
+ create_pagetables();
+ *firstaddr = avail_start;
+
+ virtual_avail = (vm_offset_t) KERNBASE + avail_start;
virtual_avail = pmap_kmem_choose(virtual_avail);
virtual_end = VM_MAX_KERNEL_ADDRESS;
+
+ /* XXX do %cr0 as well */
+ load_cr4(rcr4() | CR4_PGE | CR4_PSE);
+ load_cr3(KPML4phys);
+
/*
* Initialize protection array.
*/
- i386_protection_init();
+ amd64_protection_init();
/*
* Initialize the kernel pmap (which is statically allocated).
*/
- kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD);
- kernel_pmap->pm_pdp = (pdp_entry_t *) (KERNBASE + IdlePDP);
- kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + IdlePML4);
+ kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
kernel_pmap->pm_active = -1; /* don't allow deactivation */
TAILQ_INIT(&kernel_pmap->pm_pvlist);
LIST_INIT(&allpmaps);
@@ -316,15 +490,10 @@ pmap_bootstrap(firstaddr, loadaddr)
va = virtual_avail;
pte = vtopte(va);
- /*
- * CMAP1/CMAP2 are used for zeroing and copying pages.
- * CMAP3 is used for the idle process page zeroing.
+ /*
+ * CMAP1 is only used for the memory test.
*/
SYSMAP(caddr_t, CMAP1, CADDR1, 1)
- SYSMAP(caddr_t, CMAP2, CADDR2, 1)
- SYSMAP(caddr_t, CMAP3, CADDR3, 1)
-
- mtx_init(&CMAPCADDR12_lock, "CMAPCADDR12", NULL, MTX_DEF);
/*
* Crashdump maps.
@@ -344,16 +513,9 @@ pmap_bootstrap(firstaddr, loadaddr)
SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
atop(round_page(MSGBUF_SIZE)))
- /*
- * ptemap is used for pmap_pte_quick
- */
- SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1);
-
virtual_avail = va;
- *CMAP1 = *CMAP2 = 0;
- for (i = 0; i < NKPT; i++)
- PTD[i] = 0;
+ *CMAP1 = 0;
invltlb();
}
@@ -365,6 +527,52 @@ pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
return (void *)kmem_alloc(kernel_map, bytes);
}
+void *
+uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
+{
+ static vm_pindex_t colour;
+ vm_page_t m;
+ int pflags;
+ void *va;
+
+ *flags = UMA_SLAB_PRIV;
+
+ if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
+ pflags = VM_ALLOC_INTERRUPT;
+ else
+ pflags = VM_ALLOC_SYSTEM;
+
+ if (wait & M_ZERO)
+ pflags |= VM_ALLOC_ZERO;
+
+ for (;;) {
+ m = vm_page_alloc(NULL, colour++, pflags | VM_ALLOC_NOOBJ);
+ if (m == NULL) {
+ if (wait & M_NOWAIT)
+ return (NULL);
+ else
+ VM_WAIT;
+ } else
+ break;
+ }
+
+ va = (void *)PHYS_TO_DMAP(m->phys_addr);
+ if ((m->flags & PG_ZERO) == 0)
+ pagezero(va);
+ return (va);
+}
+
+void
+uma_small_free(void *mem, int size, u_int8_t flags)
+{
+ vm_page_t m;
+
+ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)mem));
+ vm_page_lock_queues();
+ vm_page_free(m);
+ vm_page_unlock_queues();
+}
+
/*
* Initialize the pmap module.
* Called by vm_init, to initialize any structures that the pmap
@@ -502,39 +710,7 @@ static __inline int
pmap_is_current(pmap_t pmap)
{
return (pmap == kernel_pmap ||
- (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME));
-}
-
-/*
- * Super fast pmap_pte routine best used when scanning
- * the pv lists. This eliminates many coarse-grained
- * invltlb calls. Note that many of the pv list
- * scans are across different pmaps. It is very wasteful
- * to do an entire invltlb for checking a single mapping.
- */
-pt_entry_t *
-pmap_pte_quick(pmap, va)
- register pmap_t pmap;
- vm_offset_t va;
-{
- pd_entry_t newpf;
- pd_entry_t *pde;
-
- pde = pmap_pde(pmap, va);
- if (*pde & PG_PS)
- return (pde);
- if (*pde != 0) {
- /* are we current address space or kernel? */
- if (pmap_is_current(pmap))
- return vtopte(va);
- newpf = *pde & PG_FRAME;
- if (((*PMAP1) & PG_FRAME) != newpf) {
- *PMAP1 = newpf | PG_RW | PG_V;
- pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR1);
- }
- return PADDR1 + (amd64_btop(va) & (NPTEPG - 1));
- }
- return (0);
+ (pmap->pm_pml4[PML4PML4I] & PG_FRAME) == (PML4pml4e[0] & PG_FRAME));
}
/*
@@ -550,24 +726,47 @@ pmap_extract(pmap, va)
{
vm_paddr_t rtval;
pt_entry_t *pte;
- pd_entry_t pde;
+ pd_entry_t pde, *pdep;
if (pmap == 0)
return 0;
- pde = pmap->pm_pdir[va >> PDRSHIFT];
- if (pde != 0) {
- if ((pde & PG_PS) != 0) {
- rtval = (pde & ~PDRMASK) | (va & PDRMASK);
+ pdep = pmap_pde(pmap, va);
+ if (pdep) {
+ pde = *pdep;
+ if (pde) {
+ if ((pde & PG_PS) != 0) {
+ rtval = (pde & ~PDRMASK) | (va & PDRMASK);
+ return rtval;
+ }
+ pte = pmap_pte(pmap, va);
+ rtval = ((*pte & PG_FRAME) | (va & PAGE_MASK));
return rtval;
}
- pte = pmap_pte_quick(pmap, va);
- rtval = ((*pte & PG_FRAME) | (va & PAGE_MASK));
- return rtval;
}
return 0;
}
+vm_paddr_t
+pmap_kextract(vm_offset_t va)
+{
+ pd_entry_t *pde;
+ vm_paddr_t pa;
+
+ if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
+ pa = DMAP_TO_PHYS(va);
+ } else {
+ pde = pmap_pde(kernel_pmap, va);
+ if (*pde & PG_PS) {
+ pa = (*pde & ~(NBPDR - 1)) | (va & (NBPDR - 1));
+ } else {
+ pa = *vtopte(va);
+ pa = (pa & PG_FRAME) | (va & PAGE_MASK);
+ }
+ }
+ return pa;
+}
+
/***************************************************
* Low level mapping routines.....
***************************************************/
@@ -689,7 +888,7 @@ retry:
#endif
/*
- * Create the kernel stack (including pcb for i386) for a new thread.
+ * Create the kernel stack (including pcb for amd64) for a new thread.
* This routine directly affects the fork perf for a process and
* create performance for a thread.
*/
@@ -899,7 +1098,7 @@ pmap_swapin_thread(td)
* drops to zero, then it decrements the wire count.
*/
static int
-_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
+_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
{
while (vm_page_sleep_if_busy(m, FALSE, "pmuwpt"))
@@ -910,14 +1109,51 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
/*
* unmap the page table page
*/
- pmap->pm_pdir[m->pindex] = 0;
+ if (m->pindex >= (NUPDE + NUPDPE)) {
+ /* PDP page */
+ pml4_entry_t *pml4;
+ pml4 = pmap_pml4e(pmap, va);
+ pteva = (vm_offset_t) PDPmap + amd64_ptob(m->pindex - (NUPDE + NUPDPE));
+ *pml4 = 0;
+ } else if (m->pindex >= NUPDE) {
+ /* PD page */
+ pdp_entry_t *pdp;
+ pdp = pmap_pdpe(pmap, va);
+ pteva = (vm_offset_t) PDmap + amd64_ptob(m->pindex - NUPDE);
+ *pdp = 0;
+ } else {
+ /* PTE page */
+ pd_entry_t *pd;
+ pd = pmap_pde(pmap, va);
+ pteva = (vm_offset_t) PTmap + amd64_ptob(m->pindex);
+ *pd = 0;
+ }
--pmap->pm_stats.resident_count;
+ if (m->pindex < NUPDE) {
+ /* Unhold the PD page */
+ vm_page_t pdpg;
+ pdpg = vm_page_lookup(pmap->pm_pteobj, NUPDE + pmap_pdpe_index(va));
+ while (vm_page_sleep_if_busy(pdpg, FALSE, "pulook"))
+ vm_page_lock_queues();
+ vm_page_unhold(pdpg);
+ if (pdpg->hold_count == 0)
+ _pmap_unwire_pte_hold(pmap, va, pdpg);
+ }
+ if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
+ /* Unhold the PDP page */
+ vm_page_t pdppg;
+ pdppg = vm_page_lookup(pmap->pm_pteobj, NUPDE + NUPDPE + pmap_pml4e_index(va));
+ while (vm_page_sleep_if_busy(pdppg, FALSE, "pulooK"))
+ vm_page_lock_queues();
+ vm_page_unhold(pdppg);
+ if (pdppg->hold_count == 0)
+ _pmap_unwire_pte_hold(pmap, va, pdppg);
+ }
if (pmap_is_current(pmap)) {
/*
* Do an invltlb to make the invalidated mapping
* take effect immediately.
*/
- pteva = VM_MAXUSER_ADDRESS + amd64_ptob(m->pindex);
pmap_invalidate_page(pmap, pteva);
}
@@ -936,11 +1172,11 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
}
static PMAP_INLINE int
-pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
+pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
{
vm_page_unhold(m);
if (m->hold_count == 0)
- return _pmap_unwire_pte_hold(pmap, m);
+ return _pmap_unwire_pte_hold(pmap, va, m);
else
return 0;
}
@@ -952,23 +1188,24 @@ pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
static int
pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte)
{
- unsigned ptepindex;
+ vm_pindex_t ptepindex;
+
if (va >= VM_MAXUSER_ADDRESS)
return 0;
if (mpte == NULL) {
- ptepindex = (va >> PDRSHIFT);
+ ptepindex = pmap_pde_pindex(va);
if (pmap->pm_pteobj->root &&
- (pmap->pm_pteobj->root->pindex == ptepindex)) {
+ pmap->pm_pteobj->root->pindex == ptepindex) {
mpte = pmap->pm_pteobj->root;
} else {
while ((mpte = vm_page_lookup(pmap->pm_pteobj, ptepindex)) != NULL &&
- vm_page_sleep_if_busy(mpte, FALSE, "pulook"))
+ vm_page_sleep_if_busy(mpte, FALSE, "pulook"))
vm_page_lock_queues();
}
}
- return pmap_unwire_pte_hold(pmap, mpte);
+ return pmap_unwire_pte_hold(pmap, va, mpte);
}
void
@@ -976,9 +1213,7 @@ pmap_pinit0(pmap)
struct pmap *pmap;
{
- pmap->pm_pdir = (pd_entry_t *)(KERNBASE + IdlePTD);
- pmap->pm_pdp = (pdp_entry_t *)(KERNBASE + IdlePDP);
- pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + IdlePML4);
+ pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvlist);
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -995,90 +1230,39 @@ void
pmap_pinit(pmap)
register struct pmap *pmap;
{
- vm_page_t ptdpg[NPGPTD];
- vm_page_t pdppg;
vm_page_t pml4pg;
- vm_paddr_t pa;
- int i;
-
- /*
- * No need to allocate page table space yet but we do need a valid
- * page directory table.
- */
- if (pmap->pm_pdir == NULL) {
- pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map,
- NBPTD);
- pmap->pm_pdp = (pdp_entry_t *)kmem_alloc_pageable(kernel_map,
- PAGE_SIZE);
- pmap->pm_pml4 = (pml4_entry_t *)kmem_alloc_pageable(kernel_map,
- PAGE_SIZE);
- }
/*
* allocate object for the ptes
*/
if (pmap->pm_pteobj == NULL)
- pmap->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, PTDPTDI +
- NPGPTD + 2);
+ pmap->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, NUPDE + NUPDPE + NUPML4E + 1);
/*
- * allocate the page directory page(s)
+ * allocate the page directory page
*/
- for (i = 0; i < NPGPTD; i++) {
- ptdpg[i] = vm_page_grab(pmap->pm_pteobj, PTDPTDI + i,
- VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED |
- VM_ALLOC_ZERO);
- vm_page_lock_queues();
- vm_page_flag_clear(ptdpg[i], PG_BUSY);
- ptdpg[i]->valid = VM_PAGE_BITS_ALL;
- vm_page_unlock_queues();
- }
-
- pml4pg = vm_page_grab(pmap->pm_pteobj, PTDPTDI + NPGPTD,
+ pml4pg = vm_page_grab(pmap->pm_pteobj, NUPDE + NUPDPE + NUPML4E,
VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
vm_page_lock_queues();
vm_page_flag_clear(pml4pg, PG_BUSY);
pml4pg->valid = VM_PAGE_BITS_ALL;
vm_page_unlock_queues();
- pdppg = vm_page_grab(pmap->pm_pteobj, PTDPTDI + NPGPTD + 1,
- VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
- vm_page_lock_queues();
- vm_page_flag_clear(pdppg, PG_BUSY);
- pdppg->valid = VM_PAGE_BITS_ALL;
- vm_page_unlock_queues();
-
- pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD);
- pmap_qenter((vm_offset_t)pmap->pm_pdp, &pdppg, 1);
- pmap_qenter((vm_offset_t)pmap->pm_pml4, &pml4pg, 1);
+ pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
- for (i = 0; i < NPGPTD; i++) {
- if ((ptdpg[i]->flags & PG_ZERO) == 0)
- bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE);
- }
- if ((pdppg->flags & PG_ZERO) == 0)
- bzero(pmap->pm_pdp, PAGE_SIZE);
if ((pml4pg->flags & PG_ZERO) == 0)
bzero(pmap->pm_pml4, PAGE_SIZE);
mtx_lock_spin(&allpmaps_lock);
LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);
mtx_unlock_spin(&allpmaps_lock);
+
/* Wire in kernel global address entries. */
- bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t));
+ pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
+ pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
/* install self-referential address mapping entry(s) */
- for (i = 0; i < NPGPTD; i++) {
- pa = VM_PAGE_TO_PHYS(ptdpg[i]);
- pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M;
- }
- /* Connect ptd pages to pdp */
- for (i = 0; i < NPGPTD; i++) {
- pa = VM_PAGE_TO_PHYS(ptdpg[i]);
- pmap->pm_pdp[i] = pa | PG_RW | PG_V | PG_U;
- }
- /* connect pdp to pml4 */
- pmap->pm_pml4[0] = VM_PAGE_TO_PHYS(pdppg) | PG_RW | PG_V | PG_U;
+ pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
pmap->pm_active = 0;
TAILQ_INIT(&pmap->pm_pvlist);
@@ -1105,11 +1289,9 @@ pmap_pinit2(pmap)
static vm_page_t
_pmap_allocpte(pmap, ptepindex)
pmap_t pmap;
- unsigned ptepindex;
+ vm_pindex_t ptepindex;
{
- vm_paddr_t ptepa;
- vm_offset_t pteva;
- vm_page_t m;
+ vm_page_t m, pdppg, pdpg;
/*
* Find or fabricate a new pagetable page
@@ -1133,22 +1315,83 @@ _pmap_allocpte(pmap, ptepindex)
pmap->pm_stats.resident_count++;
- ptepa = VM_PAGE_TO_PHYS(m);
- pmap->pm_pdir[ptepindex] =
- (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
+ if (ptepindex >= (NUPDE + NUPDPE)) {
+ pml4_entry_t *pml4;
+ vm_pindex_t pml4index;
+
+ /* Wire up a new PDPE page */
+ pml4index = ptepindex - (NUPDE + NUPDPE);
+ pml4 = &pmap->pm_pml4[pml4index];
+ *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
+
+ } else if (ptepindex >= NUPDE) {
+ vm_pindex_t pml4index;
+ vm_pindex_t pdpindex;
+ pml4_entry_t *pml4;
+ pdp_entry_t *pdp;
+
+ /* Wire up a new PDE page */
+ pdpindex = ptepindex - NUPDE;
+ pml4index = pdpindex >> NPML4EPGSHIFT;
+
+ pml4 = &pmap->pm_pml4[pml4index];
+ if ((*pml4 & PG_V) == 0) {
+ /* Have to allocate a new pdp, recurse */
+ _pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index);
+ } else {
+ /* Add reference to pdp page */
+ pdppg = pmap_page_lookup(pmap->pm_pteobj, NUPDE + NUPDPE + pml4index);
+ pdppg->hold_count++;
+ }
+ pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
+
+ /* Now find the pdp page */
+ pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
+ *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
+
+ } else {
+ vm_pindex_t pml4index;
+ vm_pindex_t pdpindex;
+ pml4_entry_t *pml4;
+ pdp_entry_t *pdp;
+ pd_entry_t *pd;
+
+ /* Wire up a new PTE page */
+ pdpindex = ptepindex >> NPDPEPGSHIFT;
+ pml4index = pdpindex >> NPML4EPGSHIFT;
+
+ /* First, find the pdp and check that its valid. */
+ pml4 = &pmap->pm_pml4[pml4index];
+ if ((*pml4 & PG_V) == 0) {
+ /* Have to allocate a new pd, recurse */
+ _pmap_allocpte(pmap, NUPDE + pdpindex);
+ pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
+ pdp = &pdp[pdpindex];
+ } else {
+ pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
+ pdp = &pdp[pdpindex];
+ if ((*pdp & PG_V) == 0) {
+ /* Have to allocate a new pd, recurse */
+ _pmap_allocpte(pmap, NUPDE + pdpindex);
+ } else {
+ /* Add reference to the pd page */
+ pdpg = pmap_page_lookup(pmap->pm_pteobj, NUPDE + pdpindex);
+ pdpg->hold_count++;
+ }
+ }
+ pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
+
+ /* Now we know where the page directory page is */
+ pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
+ *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
+ }
/*
* Try to use the new mapping, but if we cannot, then
* do it with the routine that maps the page explicitly.
*/
- if ((m->flags & PG_ZERO) == 0) {
- if (pmap_is_current(pmap)) {
- pteva = VM_MAXUSER_ADDRESS + amd64_ptob(ptepindex);
- bzero((caddr_t) pteva, PAGE_SIZE);
- } else {
- pmap_zero_page(m);
- }
- }
+ if ((m->flags & PG_ZERO) == 0)
+ pmap_zero_page(m);
vm_page_lock_queues();
m->valid = VM_PAGE_BITS_ALL;
vm_page_flag_clear(m, PG_ZERO);
@@ -1161,27 +1404,27 @@ _pmap_allocpte(pmap, ptepindex)
static vm_page_t
pmap_allocpte(pmap_t pmap, vm_offset_t va)
{
- unsigned ptepindex;
- pd_entry_t ptepa;
+ vm_pindex_t ptepindex;
+ pd_entry_t *pd;
vm_page_t m;
/*
* Calculate pagetable page index
*/
- ptepindex = va >> PDRSHIFT;
+ ptepindex = pmap_pde_pindex(va);
/*
* Get the page directory entry
*/
- ptepa = pmap->pm_pdir[ptepindex];
+ pd = pmap_pde(pmap, va);
/*
* This supports switching from a 2MB page to a
* normal 4K page.
*/
- if (ptepa & PG_PS) {
- pmap->pm_pdir[ptepindex] = 0;
- ptepa = 0;
+ if (pd != 0 && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
+ *pd = 0;
+ pd = 0;
pmap_invalidate_all(kernel_pmap);
}
@@ -1189,7 +1432,7 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va)
* If the page table page is mapped, we just increment the
* hold count, and activate it.
*/
- if (ptepa) {
+ if (pd != 0 && (*pd & PG_V) != 0) {
/*
* In order to get the page table page, try the
* hint first.
@@ -1206,7 +1449,8 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va)
/*
* Here if the pte page isn't mapped, or if it has been deallocated.
*/
- return _pmap_allocpte(pmap, ptepindex);
+ m = _pmap_allocpte(pmap, ptepindex);
+ return m;
}
@@ -1224,7 +1468,6 @@ pmap_release(pmap_t pmap)
{
vm_object_t object;
vm_page_t m;
- int i;
object = pmap->pm_pteobj;
@@ -1239,24 +1482,8 @@ pmap_release(pmap_t pmap)
LIST_REMOVE(pmap, pm_list);
mtx_unlock_spin(&allpmaps_lock);
- bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) *
- sizeof(*pmap->pm_pdir));
-
- pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
-
vm_page_lock_queues();
- for (i = 0; i < NPGPTD; i++) {
- m = TAILQ_FIRST(&object->memq);
- KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdp[i] & PG_FRAME),
- ("pmap_release: got wrong ptd page"));
- m->wire_count--;
- atomic_subtract_int(&cnt.v_wire_count, 1);
- vm_page_busy(m);
- vm_page_free_zero(m);
- }
- /* now free pdp and pml4 */
- for (i = 0; i < 2; i++) {
- m = TAILQ_FIRST(&object->memq);
+ while ((m = TAILQ_FIRST(&object->memq)) != NULL) {
m->wire_count--;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_busy(m);
@@ -1293,26 +1520,24 @@ SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD,
void
pmap_growkernel(vm_offset_t addr)
{
- struct pmap *pmap;
int s;
vm_paddr_t ptppaddr;
vm_page_t nkpg;
pd_entry_t newpdir;
- pt_entry_t *pde;
s = splhigh();
mtx_assert(&kernel_map->system_mtx, MA_OWNED);
if (kernel_vm_end == 0) {
kernel_vm_end = KERNBASE;
nkpt = 0;
- while (pdir_pde(PTD, kernel_vm_end)) {
+ while ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) {
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
nkpt++;
}
}
addr = roundup2(addr, PAGE_SIZE * NPTEPG);
while (kernel_vm_end < addr) {
- if (pdir_pde(PTD, kernel_vm_end)) {
+ if ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) {
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
continue;
}
@@ -1330,14 +1555,8 @@ pmap_growkernel(vm_offset_t addr)
pmap_zero_page(nkpg);
ptppaddr = VM_PAGE_TO_PHYS(nkpg);
newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
- pdir_pde(PTD, kernel_vm_end) = newpdir;
+ *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir;
- mtx_lock_spin(&allpmaps_lock);
- LIST_FOREACH(pmap, &allpmaps, pm_list) {
- pde = pmap_pde(pmap, kernel_vm_end);
- pde_store(pde, newpdir);
- }
- mtx_unlock_spin(&allpmaps_lock);
kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
}
splx(s);
@@ -1495,7 +1714,8 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va)
{
pt_entry_t *pte;
- if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
+ pte = pmap_pte(pmap, va);
+ if (pte == NULL || (*pte & PG_V) == 0)
return;
pmap_remove_pte(pmap, pte, va);
pmap_invalidate_page(pmap, va);
@@ -1511,7 +1731,7 @@ void
pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
vm_offset_t pdnxt;
- pd_entry_t ptpaddr;
+ pd_entry_t ptpaddr, *pde;
pt_entry_t *pte;
int anyvalid;
@@ -1526,26 +1746,30 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
* common operation and easy to short circuit some
* code.
*/
- if ((sva + PAGE_SIZE == eva) &&
- ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
- pmap_remove_page(pmap, sva);
- return;
+ if (sva + PAGE_SIZE == eva) {
+ pde = pmap_pde(pmap, sva);
+ if (pde && (*pde & PG_PS) == 0) {
+ pmap_remove_page(pmap, sva);
+ return;
+ }
}
anyvalid = 0;
for (; sva < eva; sva = pdnxt) {
- unsigned pdirindex;
+
+ if (pmap->pm_stats.resident_count == 0)
+ break;
/*
* Calculate index for next page table.
*/
pdnxt = (sva + NBPDR) & ~PDRMASK;
- if (pmap->pm_stats.resident_count == 0)
- break;
- pdirindex = sva >> PDRSHIFT;
- ptpaddr = pmap->pm_pdir[pdirindex];
+ pde = pmap_pde(pmap, sva);
+ if (pde == 0)
+ continue;
+ ptpaddr = *pde;
/*
* Weed out invalid mappings. Note: we assume that the page
@@ -1558,7 +1782,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
* Check for large page.
*/
if ((ptpaddr & PG_PS) != 0) {
- pmap->pm_pdir[pdirindex] = 0;
+ *pde = 0;
pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
anyvalid = 1;
continue;
@@ -1573,8 +1797,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
pdnxt = eva;
for (; sva != pdnxt; sva += PAGE_SIZE) {
- if ((pte = pmap_pte_quick(pmap, sva)) == NULL ||
- *pte == 0)
+ pte = pmap_pte(pmap, sva);
+ if (pte == NULL || *pte == 0)
continue;
anyvalid = 1;
if (pmap_remove_pte(pmap, pte, sva))
@@ -1619,7 +1843,7 @@ pmap_remove_all(vm_page_t m)
s = splvm();
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
pv->pv_pmap->pm_stats.resident_count--;
- pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
+ pte = pmap_pte(pv->pv_pmap, pv->pv_va);
tpte = pte_load_clear(pte);
if (tpte & PG_W)
pv->pv_pmap->pm_stats.wired_count--;
@@ -1659,7 +1883,7 @@ void
pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
{
vm_offset_t pdnxt;
- pd_entry_t ptpaddr;
+ pd_entry_t ptpaddr, *pde;
int anychanged;
if (pmap == NULL)
@@ -1676,12 +1900,13 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
anychanged = 0;
for (; sva < eva; sva = pdnxt) {
- unsigned pdirindex;
pdnxt = (sva + NBPDR) & ~PDRMASK;
- pdirindex = sva >> PDRSHIFT;
- ptpaddr = pmap->pm_pdir[pdirindex];
+ pde = pmap_pde(pmap, sva);
+ if (pde == NULL)
+ continue;
+ ptpaddr = *pde;
/*
* Weed out invalid mappings. Note: we assume that the page
@@ -1694,7 +1919,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
* Check for large page.
*/
if ((ptpaddr & PG_PS) != 0) {
- pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
+ *pde &= ~(PG_M|PG_RW);
pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
anychanged = 1;
continue;
@@ -1708,7 +1933,8 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
pt_entry_t *pte;
vm_page_t m;
- if ((pte = pmap_pte_quick(pmap, sva)) == NULL)
+ pte = pmap_pte(pmap, sva);
+ if (pte == NULL)
continue;
pbits = *pte;
if (pbits & PG_MANAGED) {
@@ -1785,21 +2011,19 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
pd_entry_t *pdeaddr = pmap_pde(pmap, va);
origpte = *pdeaddr;
if ((origpte & PG_V) == 0) {
- panic("pmap_enter: invalid kernel page table page, pdir=%p, pde=%p, va=%p\n",
- pmap->pm_pdir[PTDPTDI], origpte, va);
+ panic("pmap_enter: invalid kernel page table page, pde=%p, va=%p\n",
+ origpte, va);
}
}
#endif
- pte = pmap_pte_quick(pmap, va);
+ pte = pmap_pte(pmap, va);
/*
* Page Directory table entry not valid, we need a new PT page
*/
- if (pte == NULL) {
- panic("pmap_enter: invalid page directory pdir=%#lx, va=%#lx\n",
- pmap->pm_pdir[PTDPTDI], va);
- }
+ if (pte == NULL)
+ panic("pmap_enter: invalid page directory va=%#lx\n", va);
pa = VM_PAGE_TO_PHYS(m) & PG_FRAME;
origpte = *pte;
@@ -1937,13 +2161,13 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
* resident, we are creating it here.
*/
if (va < VM_MAXUSER_ADDRESS) {
- unsigned ptepindex;
- pd_entry_t ptepa;
+ vm_pindex_t ptepindex;
+ pd_entry_t *ptepa;
/*
* Calculate pagetable page index
*/
- ptepindex = va >> PDRSHIFT;
+ ptepindex = pmap_pde_pindex(va);
if (mpte && (mpte->pindex == ptepindex)) {
mpte->hold_count++;
} else {
@@ -1951,14 +2175,14 @@ retry:
/*
* Get the page directory entry
*/
- ptepa = pmap->pm_pdir[ptepindex];
+ ptepa = pmap_pde(pmap, va);
/*
* If the page table page is mapped, we just increment
* the hold count, and activate it.
*/
- if (ptepa) {
- if (ptepa & PG_PS)
+ if (ptepa && (*ptepa & PG_V) != 0) {
+ if (*ptepa & PG_PS)
panic("pmap_enter_quick: unexpected mapping into 2MB page");
if (pmap->pm_pteobj->root &&
(pmap->pm_pteobj->root->pindex == ptepindex)) {
@@ -1980,14 +2204,14 @@ retry:
/*
* This call to vtopte makes the assumption that we are
* entering the page into the current pmap. In order to support
- * quick entry into any pmap, one would likely use pmap_pte_quick.
+ * quick entry into any pmap, one would likely use pmap_pte.
* But that isn't as quick as vtopte.
*/
pte = vtopte(va);
if (*pte) {
if (mpte != NULL) {
vm_page_lock_queues();
- pmap_unwire_pte_hold(pmap, mpte);
+ pmap_unwire_pte_hold(pmap, va, mpte);
vm_page_unlock_queues();
}
return 0;
@@ -2045,7 +2269,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
vm_object_t object, vm_pindex_t pindex,
vm_size_t size, int limit)
{
- vm_offset_t tmpidx;
+ vm_pindex_t tmpidx;
int psize;
vm_page_t p, mpte;
@@ -2061,13 +2285,12 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0)) {
int i;
vm_page_t m[1];
- unsigned int ptepindex;
int npdes;
- pd_entry_t ptepa;
+ pd_entry_t ptepa, *pde;
- if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)])
+ pde = pmap_pde(pmap, addr);
+ if (pde != 0 && (*pde & PG_V) != 0)
return;
-
retry:
p = vm_page_lookup(object, pindex);
if (p != NULL) {
@@ -2103,10 +2326,9 @@ retry:
pmap->pm_stats.resident_count += size >> PAGE_SHIFT;
npdes = size >> PDRSHIFT;
for(i = 0; i < npdes; i++) {
- pde_store(&pmap->pm_pdir[ptepindex],
- ptepa | PG_U | PG_RW | PG_V | PG_PS);
+ pde_store(pde, ptepa | PG_U | PG_RW | PG_V | PG_PS);
ptepa += NBPDR;
- ptepindex += 1;
+ pde++;
}
pmap_invalidate_all(kernel_pmap);
return;
@@ -2166,7 +2388,6 @@ retry:
}
vm_page_unlock_queues();
}
- return;
}
/*
@@ -2198,6 +2419,7 @@ pmap_prefault(pmap, addra, entry)
vm_pindex_t pindex;
vm_page_t m, mpte;
vm_object_t object;
+ pd_entry_t *pde;
if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)))
return;
@@ -2223,11 +2445,12 @@ pmap_prefault(pmap, addra, entry)
if (addr < starta || addr >= entry->end)
continue;
- if ((*pmap_pde(pmap, addr)) == 0)
+ pde = pmap_pde(pmap, addr);
+ if (pde == NULL || (*pde & PG_V) == 0)
continue;
pte = vtopte(addr);
- if (*pte)
+ if ((*pte & PG_V) == 0)
continue;
pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
@@ -2282,18 +2505,18 @@ pmap_change_wiring(pmap, va, wired)
if (pmap == NULL)
return;
- pte = pmap_pte_quick(pmap, va);
-
- if (wired && !pmap_pte_w(pte))
- pmap->pm_stats.wired_count++;
- else if (!wired && pmap_pte_w(pte))
- pmap->pm_stats.wired_count--;
-
/*
* Wiring is not a hardware characteristic so there is no need to
* invalidate TLB.
*/
- pmap_pte_set_w(pte, wired);
+ pte = pmap_pte(pmap, va);
+ if (wired && (*pte & PG_W) == 0) {
+ pmap->pm_stats.wired_count++;
+ *pte |= PG_W;
+ } else if (!wired && (*pte & PG_W) != 0) {
+ pmap->pm_stats.wired_count--;
+ *pte &= ~PG_W;
+ }
}
@@ -2324,8 +2547,8 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
for (addr = src_addr; addr < end_addr; addr = pdnxt) {
pt_entry_t *src_pte, *dst_pte;
vm_page_t dstmpte, srcmpte;
- pd_entry_t srcptepaddr;
- unsigned ptepindex;
+ pd_entry_t srcptepaddr, *pde;
+ vm_pindex_t ptepindex;
if (addr >= UPT_MIN_ADDRESS)
panic("pmap_copy: invalid to pmap_copy page tables\n");
@@ -2340,15 +2563,27 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
break;
pdnxt = (addr + NBPDR) & ~PDRMASK;
- ptepindex = addr >> PDRSHIFT;
+ ptepindex = pmap_pde_pindex(addr);
- srcptepaddr = src_pmap->pm_pdir[ptepindex];
+ pde = pmap_pde(src_pmap, addr);
+ if (pde)
+ srcptepaddr = *pde;
+ else
+ continue;
if (srcptepaddr == 0)
continue;
if (srcptepaddr & PG_PS) {
- if (dst_pmap->pm_pdir[ptepindex] == 0) {
- dst_pmap->pm_pdir[ptepindex] = srcptepaddr;
+ pde = pmap_pde(dst_pmap, addr);
+ if (pde == 0) {
+ /*
+ * XXX should do an allocpte here to
+ * instantiate the pde
+ */
+ continue;
+ }
+ if (*pde == 0) {
+ *pde = srcptepaddr;
dst_pmap->pm_stats.resident_count +=
NBPDR / PAGE_SIZE;
}
@@ -2377,7 +2612,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
* block.
*/
dstmpte = pmap_allocpte(dst_pmap, addr);
- dst_pte = pmap_pte_quick(dst_pmap, addr);
+ dst_pte = pmap_pte(dst_pmap, addr);
if ((*dst_pte == 0) && (ptetemp = *src_pte)) {
/*
* Clear the modified and
@@ -2391,7 +2626,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
dstmpte, m);
} else {
vm_page_lock_queues();
- pmap_unwire_pte_hold(dst_pmap, dstmpte);
+ pmap_unwire_pte_hold(dst_pmap, addr, dstmpte);
vm_page_unlock_queues();
}
if (dstmpte->hold_count >= srcmpte->hold_count)
@@ -2410,15 +2645,9 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
void
pmap_zero_page(vm_page_t m)
{
+ vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
- mtx_lock(&CMAPCADDR12_lock);
- if (*CMAP2)
- panic("pmap_zero_page: CMAP2 busy");
- *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
- invlpg((u_long)CADDR2);
- pagezero(CADDR2);
- *CMAP2 = 0;
- mtx_unlock(&CMAPCADDR12_lock);
+ pagezero((void *)va);
}
/*
@@ -2430,18 +2659,12 @@ pmap_zero_page(vm_page_t m)
void
pmap_zero_page_area(vm_page_t m, int off, int size)
{
+ vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
- mtx_lock(&CMAPCADDR12_lock);
- if (*CMAP2)
- panic("pmap_zero_page: CMAP2 busy");
- *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
- invlpg((u_long)CADDR2);
if (off == 0 && size == PAGE_SIZE)
- pagezero(CADDR2);
+ pagezero((void *)va);
else
- bzero((char *)CADDR2 + off, size);
- *CMAP2 = 0;
- mtx_unlock(&CMAPCADDR12_lock);
+ bzero((char *)va + off, size);
}
/*
@@ -2453,13 +2676,9 @@ pmap_zero_page_area(vm_page_t m, int off, int size)
void
pmap_zero_page_idle(vm_page_t m)
{
+ vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
- if (*CMAP3)
- panic("pmap_zero_page: CMAP3 busy");
- *CMAP3 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
- invlpg((u_long)CADDR3);
- pagezero(CADDR3);
- *CMAP3 = 0;
+ pagezero((void *)va);
}
/*
@@ -2469,22 +2688,12 @@ pmap_zero_page_idle(vm_page_t m)
* time.
*/
void
-pmap_copy_page(vm_page_t src, vm_page_t dst)
-{
-
- mtx_lock(&CMAPCADDR12_lock);
- if (*CMAP1)
- panic("pmap_copy_page: CMAP1 busy");
- if (*CMAP2)
- panic("pmap_copy_page: CMAP2 busy");
- *CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A;
- *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M;
- invlpg((u_long)CADDR1);
- invlpg((u_long)CADDR2);
- bcopy(CADDR1, CADDR2, PAGE_SIZE);
- *CMAP1 = 0;
- *CMAP2 = 0;
- mtx_unlock(&CMAPCADDR12_lock);
+pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
+{
+ vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
+ vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
+
+ bcopy((void *)src, (void *)dst, PAGE_SIZE);
}
/*
@@ -2558,7 +2767,7 @@ pmap_remove_pages(pmap, sva, eva)
#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
pte = vtopte(pv->pv_va);
#else
- pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
+ pte = pmap_pte(pv->pv_pmap, pv->pv_va);
#endif
tpte = *pte;
@@ -2643,7 +2852,7 @@ pmap_is_modified(vm_page_t m)
continue;
}
#endif
- pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
+ pte = pmap_pte(pv->pv_pmap, pv->pv_va);
if (*pte & PG_M) {
splx(s);
return TRUE;
@@ -2689,7 +2898,7 @@ pmap_changebit(vm_page_t m, int bit, boolean_t setem)
}
#endif
- pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
+ pte = pmap_pte(pv->pv_pmap, pv->pv_va);
if (setem) {
*pte |= bit;
@@ -2771,7 +2980,7 @@ pmap_ts_referenced(vm_page_t m)
if (!pmap_track_modified(pv->pv_va))
continue;
- pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
+ pte = pmap_pte(pv->pv_pmap, pv->pv_va);
if (pte && ((v = pte_load(pte)) & PG_A) != 0) {
pte_store(pte, v & ~PG_A);
@@ -2814,26 +3023,32 @@ pmap_clear_reference(vm_page_t m)
*/
static void
-i386_protection_init()
+amd64_protection_init()
{
- register int *kp, prot;
+ register long *kp, prot;
+
+#if 0
+#define PG_NX (1ul << 63)
+#else
+#define PG_NX 0
+#endif
kp = protection_codes;
for (prot = 0; prot < 8; prot++) {
switch (prot) {
case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE:
- /*
- * Read access is also 0. There isn't any execute bit,
- * so just make it readable.
- */
case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE:
+ *kp++ = PG_NX;
+ break;
case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE:
case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE:
*kp++ = 0;
break;
case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE:
- case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE:
+ *kp++ = PG_RW | PG_NX;
+ break;
+ case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE:
*kp++ = PG_RW;
break;
@@ -2905,7 +3120,7 @@ pmap_mincore(pmap, addr)
vm_page_t m;
int val = 0;
- ptep = pmap_pte_quick(pmap, addr);
+ ptep = pmap_pte(pmap, addr);
if (ptep == 0) {
return 0;
}
@@ -2994,113 +3209,3 @@ pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
return addr;
}
-
-
-#if defined(PMAP_DEBUG)
-pmap_pid_dump(int pid)
-{
- pmap_t pmap;
- struct proc *p;
- int npte = 0;
- int index;
-
- sx_slock(&allproc_lock);
- LIST_FOREACH(p, &allproc, p_list) {
- if (p->p_pid != pid)
- continue;
-
- if (p->p_vmspace) {
- int i,j;
- index = 0;
- pmap = vmspace_pmap(p->p_vmspace);
- for (i = 0; i < NPDEPTD; i++) {
- pd_entry_t *pde;
- pt_entry_t *pte;
- vm_offset_t base = i << PDRSHIFT;
-
- pde = &pmap->pm_pdir[i];
- if (pde && pmap_pde_v(pde)) {
- for (j = 0; j < NPTEPG; j++) {
- vm_offset_t va = base + (j << PAGE_SHIFT);
- if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) {
- if (index) {
- index = 0;
- printf("\n");
- }
- sx_sunlock(&allproc_lock);
- return npte;
- }
- pte = pmap_pte_quick(pmap, va);
- if (pte && pmap_pte_v(pte)) {
- pt_entry_t pa;
- vm_page_t m;
- pa = *pte;
- m = PHYS_TO_VM_PAGE(pa);
- printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x",
- va, pa, m->hold_count, m->wire_count, m->flags);
- npte++;
- index++;
- if (index >= 2) {
- index = 0;
- printf("\n");
- } else {
- printf(" ");
- }
- }
- }
- }
- }
- }
- }
- sx_sunlock(&allproc_lock);
- return npte;
-}
-#endif
-
-#if defined(DEBUG)
-
-static void pads(pmap_t pm);
-void pmap_pvdump(vm_offset_t pa);
-
-/* print address space of pmap*/
-static void
-pads(pm)
- pmap_t pm;
-{
- int i, j;
- vm_paddr_t va;
- pt_entry_t *ptep;
-
- if (pm == kernel_pmap)
- return;
- for (i = 0; i < NPDEPTD; i++)
- if (pm->pm_pdir[i])
- for (j = 0; j < NPTEPG; j++) {
- va = (i << PDRSHIFT) + (j << PAGE_SHIFT);
- if (pm == kernel_pmap && va < KERNBASE)
- continue;
- if (pm != kernel_pmap && va > UPT_MAX_ADDRESS)
- continue;
- ptep = pmap_pte_quick(pm, va);
- if (pmap_pte_v(ptep))
- printf("%x:%x ", va, *ptep);
- };
-
-}
-
-void
-pmap_pvdump(pa)
- vm_paddr_t pa;
-{
- pv_entry_t pv;
- vm_page_t m;
-
- printf("pa %x", pa);
- m = PHYS_TO_VM_PAGE(pa);
- TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
- printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va);
- pads(pv->pv_pmap);
- }
- printf(" ");
-}
-#endif
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index bc69e42..72ba590 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -212,7 +212,7 @@ trap(frame)
* kernel can print out a useful trap message and even get
* to the debugger.
*/
- if (td->td_critnest == 0)
+ if (td->td_critnest != 0)
trap_fatal(&frame, frame.tf_addr);
}
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index 0b7f510..f978290 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -264,7 +264,7 @@ cpu_reset_real()
printf("Keyboard reset did not work, attempting CPU shutdown\n");
DELAY(1000000); /* wait 1 sec for printf to complete */
/* force a shutdown by unmapping entire address space ! */
- bzero((caddr_t)PTD, NBPTD);
+ bzero((caddr_t)PML4map, PAGE_SIZE);
/* "good night, sweet prince .... <THUNK!>" */
invltlb();
diff --git a/sys/amd64/include/bus_amd64.h b/sys/amd64/include/bus_amd64.h
index 5f0e22a..71a25e4 100644
--- a/sys/amd64/include/bus_amd64.h
+++ b/sys/amd64/include/bus_amd64.h
@@ -401,7 +401,7 @@ bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
else
#endif
{
- int _port_ = bsh + offset;
+ bus_space_handle_t _port_ = bsh + offset;
#ifdef __GNUC__
__asm __volatile(" \n\
cld \n\
@@ -443,7 +443,7 @@ bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
else
#endif
{
- int _port_ = bsh + offset;
+ bus_space_handle_t _port_ = bsh + offset;
#ifdef __GNUC__
__asm __volatile(" \n\
cld \n\
@@ -485,7 +485,7 @@ bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
else
#endif
{
- int _port_ = bsh + offset;
+ bus_space_handle_t _port_ = bsh + offset;
#ifdef __GNUC__
__asm __volatile(" \n\
cld \n\
@@ -741,7 +741,7 @@ bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
else
#endif
{
- int _port_ = bsh + offset;
+ bus_space_handle_t _port_ = bsh + offset;
#ifdef __GNUC__
__asm __volatile(" \n\
cld \n\
@@ -783,7 +783,7 @@ bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
else
#endif
{
- int _port_ = bsh + offset;
+ bus_space_handle_t _port_ = bsh + offset;
#ifdef __GNUC__
__asm __volatile(" \n\
cld \n\
@@ -825,7 +825,7 @@ bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
else
#endif
{
- int _port_ = bsh + offset;
+ bus_space_handle_t _port_ = bsh + offset;
#ifdef __GNUC__
__asm __volatile(" \n\
cld \n\
diff --git a/sys/amd64/include/param.h b/sys/amd64/include/param.h
index 355e95d..9f73c2c 100644
--- a/sys/amd64/include/param.h
+++ b/sys/amd64/include/param.h
@@ -89,34 +89,32 @@
#define ALIGN(p) _ALIGN(p)
#define ALIGNED_POINTER(p,t) _ALIGNED_POINTER((p),(t))
+
/* Size of the level 1 page table units */
#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t)))
+#define NPTEPGSHIFT 9 /* LOG2(NPTEPG) */
#define PAGE_SHIFT 12 /* LOG2(PAGE_SIZE) */
#define PAGE_SIZE (1<<PAGE_SHIFT) /* bytes/page */
#define PAGE_MASK (PAGE_SIZE-1)
/* Size of the level 2 page directory units */
#define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t)))
+#define NPDEPGSHIFT 9 /* LOG2(NPDEPG) */
#define PDRSHIFT 21 /* LOG2(NBPDR) */
#define NBPDR (1<<PDRSHIFT) /* bytes/page dir */
#define PDRMASK (NBPDR-1)
/* Size of the level 3 page directory pointer table units */
#define NPDPEPG (PAGE_SIZE/(sizeof (pdp_entry_t)))
+#define NPDPEPGSHIFT 9 /* LOG2(NPDPEPG) */
#define PDPSHIFT 30 /* LOG2(NBPDP) */
#define NBPDP (1<<PDPSHIFT) /* bytes/page dir ptr table */
#define PDPMASK (NBPDP-1)
/* Size of the level 4 page-map level-4 table units */
#define NPML4EPG (PAGE_SIZE/(sizeof (pml4_entry_t)))
+#define NPML4EPGSHIFT 9 /* LOG2(NPML4EPG) */
#define PML4SHIFT 39 /* LOG2(NBPML4T) */
#define NBPML4T (1ul<<PML4SHIFT)/* bytes/page map lev4 table */
#define PML4MASK (NBPML4T-1)
-#define NKPML4E 1 /* addressable number of page tables/pde's */
-#define NKPDPE 1 /* addressable number of page tables/pde's */
-#define NPGPTD 4
-
-#define NBPTD (NPGPTD<<PAGE_SHIFT)
-#define NPDEPTD (NBPTD/(sizeof (pd_entry_t)))
-
#define IOPAGES 2 /* pages of i/o permission bitmap */
#define KSTACK_PAGES 4 /* pages of kstack (with pcb) */
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index 5ffa054..ef02ac1 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -83,13 +83,6 @@
#define PGEX_U 0x04 /* access from User mode (UPL) */
/*
- * Size of Kernel address space. This is the number of level 4 (top)
- * entries. We use half of them for the kernel due to the 48 bit
- * virtual address sign extension.
- */
-#define KVA_PAGES 1536
-
-/*
* Pte related macros. This is complicated by having to deal with
* the sign extension of the 48th bit.
*/
@@ -105,15 +98,26 @@
#ifndef NKPT
#define NKPT 120 /* initial number of kernel page tables */
#endif
-#ifndef NKPDE
-#define NKPDE (KVA_PAGES) /* number of page tables/pde's */
-#endif
+
+#define NKPML4E 1 /* number of kernel PML4 slots */
+#define NKPDPE 1 /* number of kernel PDP slots */
+#define NKPDE (NKPDPE*NPDEPG) /* number of kernel PD slots */
+
+#define NUPML4E 1 /* number of userland PML4 pages */
+#define NUPDPE (NUPML4E*NPDPEPG)/* number of userland PDP pages */
+#define NUPDE (NUPDPE*NPDEPG) /* number of userland PD entries */
+
+#define NDMPML4E 1 /* number of dmap PML4 slots */
/*
- * The *PTDI values control the layout of virtual memory
+ * The *PDI values control the layout of virtual memory
*/
-#define KPTDI (NPDEPTD-NKPDE) /* start of kernel virtual pde's */
-#define PTDPTDI (KPTDI-NPGPTD) /* ptd entry that points to ptd! */
+#define PML4PML4I (NPML4EPG/2) /* Index of recursive pml4 mapping */
+
+#define KPML4I (NPML4EPG-1)
+#define DMPML4I (KPML4I-1)
+
+#define KPDPI (NPDPEPG-1)
/*
* XXX doesn't really belong here I guess...
@@ -145,13 +149,18 @@ typedef u_int64_t pml4_entry_t;
* in the page tables and the evil overlapping.
*/
#ifdef _KERNEL
-#define PTmap ((pt_entry_t *)(VADDR(0, 0, PTDPTDI, 0)))
-#define PTD ((pd_entry_t *)(VADDR(0, 0, PTDPTDI, PTDPTDI)))
-#define PTDpde ((pd_entry_t *)(VADDR(0, 0, PTDPTDI, PTDPTDI) + (PTDPTDI * sizeof(pd_entry_t))))
-
-extern u_int64_t IdlePML4; /* physical address of "Idle" state directory */
-extern u_int64_t IdlePDP; /* physical address of "Idle" state directory */
-extern u_int64_t IdlePTD; /* physical address of "Idle" state directory */
+#define addr_PTmap (VADDR(PML4PML4I, 0, 0, 0))
+#define addr_PDmap (VADDR(PML4PML4I, PML4PML4I, 0, 0))
+#define addr_PDPmap (VADDR(PML4PML4I, PML4PML4I, PML4PML4I, 0))
+#define addr_PML4map (VADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I))
+#define addr_PML4pml4e (addr_PML4map + (PML4PML4I * sizeof(pml4_entry_t)))
+#define PTmap ((pt_entry_t *)(addr_PTmap))
+#define PDmap ((pd_entry_t *)(addr_PDmap))
+#define PDPmap ((pd_entry_t *)(addr_PDPmap))
+#define PML4map ((pd_entry_t *)(addr_PML4map))
+#define PML4pml4e ((pd_entry_t *)(addr_PML4pml4e))
+
+extern u_int64_t KPML4phys; /* physical address of kernel level 4 */
#endif
#ifdef _KERNEL
@@ -161,28 +170,8 @@ extern u_int64_t IdlePTD; /* physical address of "Idle" state directory */
* Note: these work recursively, thus vtopte of a pte will give
* the corresponding pde that in turn maps it.
*/
-#define vtopte(va) (PTmap + amd64_btop(va))
-
-/*
- * Routine: pmap_kextract
- * Function:
- * Extract the physical page address associated
- * kernel virtual address.
- */
-static __inline vm_paddr_t
-pmap_kextract(vm_offset_t va)
-{
- vm_paddr_t pa;
-
- pa = PTD[va >> PDRSHIFT];
- if (pa & PG_PS) {
- pa = (pa & ~(NBPDR - 1)) | (va & (NBPDR - 1));
- } else {
- pa = *vtopte(va);
- pa = (pa & PG_FRAME) | (va & PAGE_MASK);
- }
- return pa;
-}
+pt_entry_t *vtopte(vm_offset_t);
+vm_paddr_t pmap_kextract(vm_offset_t);
#define vtophys(va) pmap_kextract(((vm_offset_t) (va)))
@@ -225,14 +214,12 @@ struct md_page {
};
struct pmap {
- pd_entry_t *pm_pdir; /* KVA of page directory */
+ pml4_entry_t *pm_pml4; /* KVA of level 4 page table */
vm_object_t pm_pteobj; /* Container for pte's */
TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */
u_long pm_active; /* active on cpus */
struct pmap_statistics pm_stats; /* pmap statistics */
LIST_ENTRY(pmap) pm_list; /* List of all pmaps */
- pdp_entry_t *pm_pdp; /* KVA of level 3 page table */
- pml4_entry_t *pm_pml4; /* KVA of level 4 page table */
};
#define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list))
@@ -278,7 +265,7 @@ extern char *ptvmmap; /* poor name! */
extern vm_offset_t virtual_avail;
extern vm_offset_t virtual_end;
-void pmap_bootstrap(vm_paddr_t, vm_paddr_t);
+void pmap_bootstrap(vm_paddr_t *);
void pmap_kenter(vm_offset_t va, vm_paddr_t pa);
void pmap_kremove(vm_offset_t);
void *pmap_mapdev(vm_paddr_t, vm_size_t);
diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h
index 0e9697d..91a62ed 100644
--- a/sys/amd64/include/vmparam.h
+++ b/sys/amd64/include/vmparam.h
@@ -41,7 +41,7 @@
#ifndef _MACHINE_VMPARAM_H_
-#define _MACHINE_VMPARAM_H_ 1
+#define _MACHINE_VMPARAM_H_ 1
/*
* Machine dependent constants for AMD64.
@@ -64,7 +64,7 @@
#define MAXSSIZ (64UL*1024*1024) /* max stack size */
#endif
#ifndef SGROWSIZ
-#define SGROWSIZ (128UL*1024) /* amount to grow stack */
+#define SGROWSIZ (128UL*1024) /* amount to grow stack */
#endif
/*
@@ -78,6 +78,12 @@
*/
#define MAXSLP 20
+/*
+ * We provide a machine specific single page allocator through the tuse
+ * of the direct mapped segment. This uses 2MB pages for reduced
+ * TLB pressure.
+ */
+#define UMA_MD_SMALL_ALLOC
/*
* Virtual addresses of things. Derived from the page directory and
@@ -86,24 +92,30 @@
* messy at times, but hey, we'll do anything to save a page :-)
*/
-#define VM_MAX_KERNEL_ADDRESS VADDR(0, 0, KPTDI+NKPDE-1, NPTEPG-1)
-#define VM_MIN_KERNEL_ADDRESS VADDR(0, 0, PTDPTDI, PTDPTDI)
+#define VM_MAX_KERNEL_ADDRESS VADDR(KPML4I, NPDPEPG-1, NKPDE-1, NPTEPG-1)
+#define VM_MIN_KERNEL_ADDRESS VADDR(KPML4I, KPDPI, 0, 0)
+
+#define DMAP_MIN_ADDRESS VADDR(DMPML4I, 0, 0, 0)
+#define DMAP_MAX_ADDRESS VADDR(DMPML4I+1, 0, 0, 0)
+
+#define KERNBASE VADDR(KPML4I, KPDPI, 0, 0)
-#define KERNBASE VADDR(0, 0, KPTDI, 0)
+#define UPT_MAX_ADDRESS VADDR(PML4PML4I, PML4PML4I, PML4PML4I, PML4PML4I)
+#define UPT_MIN_ADDRESS VADDR(PML4PML4I, 0, 0, 0)
-#define UPT_MAX_ADDRESS VADDR(0, 0, PTDPTDI, PTDPTDI)
-#define UPT_MIN_ADDRESS VADDR(0, 0, PTDPTDI, 0)
+#define VM_MAXUSER_ADDRESS VADDR(NUPML4E, 0, 0, 0)
-#define VM_MAXUSER_ADDRESS UPT_MIN_ADDRESS
+#define USRSTACK VM_MAXUSER_ADDRESS
-#define USRSTACK VM_MAXUSER_ADDRESS
+#define VM_MAX_ADDRESS UPT_MAX_ADDRESS
+#define VM_MIN_ADDRESS (0)
-#define VM_MAX_ADDRESS UPT_MAX_ADDRESS
-#define VM_MIN_ADDRESS (0)
+#define PHYS_TO_DMAP(x) ((x) | DMAP_MIN_ADDRESS)
+#define DMAP_TO_PHYS(x) ((x) & ~DMAP_MIN_ADDRESS)
/* virtual sizes (bytes) for various kernel submaps */
#ifndef VM_KMEM_SIZE
-#define VM_KMEM_SIZE (12 * 1024 * 1024)
+#define VM_KMEM_SIZE (12 * 1024 * 1024)
#endif
/*
diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk
index ba85101..253d267 100644
--- a/sys/conf/kern.mk
+++ b/sys/conf/kern.mk
@@ -57,7 +57,7 @@ CFLAGS+= -mcmodel=medlow -msoft-float
# once pmap is ready. Be excessively careful to not generate FPU code.
#
.if ${MACHINE_ARCH} == "amd64"
-CFLAGS+= -mcmodel=medium -mno-red-zone \
+CFLAGS+= -mcmodel=kernel -mno-red-zone \
-mfpmath=387 -mno-sse -mno-sse2 -mno-mmx -mno-3dnow -msoft-float
.endif
OpenPOWER on IntegriCloud