diff options
Diffstat (limited to 'sys/i386')
| -rw-r--r-- | sys/i386/i386/genassym.c | 3 | ||||
| -rw-r--r-- | sys/i386/i386/locore.s | 4 | ||||
| -rw-r--r-- | sys/i386/i386/machdep.c | 77 | ||||
| -rw-r--r-- | sys/i386/i386/mp_machdep.c | 11 | ||||
| -rw-r--r-- | sys/i386/i386/pmap.c | 183 | ||||
| -rw-r--r-- | sys/i386/i386/stack_machdep.c | 87 | ||||
| -rw-r--r-- | sys/i386/i386/sys_machdep.c | 2 | ||||
| -rw-r--r-- | sys/i386/i386/trap.c | 65 | ||||
| -rw-r--r-- | sys/i386/i386/vm_machdep.c | 4 | ||||
| -rw-r--r-- | sys/i386/include/atomic.h | 95 | ||||
| -rw-r--r-- | sys/i386/include/in_cksum.h | 2 | ||||
| -rw-r--r-- | sys/i386/include/md_var.h | 1 | ||||
| -rw-r--r-- | sys/i386/include/param.h | 5 | ||||
| -rw-r--r-- | sys/i386/include/pcpu.h | 3 | ||||
| -rw-r--r-- | sys/i386/include/privatespace.h | 49 | ||||
| -rw-r--r-- | sys/i386/include/stack.h | 42 | ||||
| -rw-r--r-- | sys/i386/include/xen/xenfunc.h | 81 | ||||
| -rw-r--r-- | sys/i386/include/xen/xenvar.h | 36 |
18 files changed, 255 insertions, 495 deletions
diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 7a00740..3087834 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -101,8 +101,7 @@ ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap)); ASSYM(V_SYSCALL, offsetof(struct vmmeter, v_syscall)); ASSYM(V_INTR, offsetof(struct vmmeter, v_intr)); -/* ASSYM(UPAGES, UPAGES);*/ -ASSYM(KSTACK_PAGES, KSTACK_PAGES); +ASSYM(TD0_KSTACK_PAGES, TD0_KSTACK_PAGES); ASSYM(PAGE_SIZE, PAGE_SIZE); ASSYM(NPTEPG, NPTEPG); ASSYM(NPDEPG, NPDEPG); diff --git a/sys/i386/i386/locore.s b/sys/i386/i386/locore.s index 5bf7944..4d8e22f 100644 --- a/sys/i386/i386/locore.s +++ b/sys/i386/i386/locore.s @@ -731,7 +731,7 @@ no_kernend: movl %esi,R(IdlePTD) /* Allocate KSTACK */ - ALLOCPAGES(KSTACK_PAGES) + ALLOCPAGES(TD0_KSTACK_PAGES) movl %esi,R(p0kpa) addl $KERNBASE, %esi movl %esi, R(proc0kstack) @@ -800,7 +800,7 @@ no_kernend: /* Map proc0's KSTACK in the physical way ... */ movl R(p0kpa), %eax - movl $(KSTACK_PAGES), %ecx + movl $(TD0_KSTACK_PAGES), %ecx fillkptphys($PG_RW) /* Map ISA hole */ diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 2be5dbc..b219d9d 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -2097,9 +2097,8 @@ getmemsize(int first) kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf32 kernel"); - if (kmdp != NULL) - smapbase = (struct bios_smap *)preload_search_info(kmdp, - MODINFO_METADATA | MODINFOMD_SMAP); + smapbase = (struct bios_smap *)preload_search_info(kmdp, + MODINFO_METADATA | MODINFOMD_SMAP); if (smapbase != NULL) { add_smap_entries(smapbase, physmap, &physmap_idx); has_smap = 1; @@ -2445,7 +2444,7 @@ init386(first) #endif thread0.td_kstack = proc0kstack; - thread0.td_kstack_pages = KSTACK_PAGES; + thread0.td_kstack_pages = TD0_KSTACK_PAGES; /* * This may be done better later if it gets more high level @@ -2612,6 +2611,40 @@ init386(first) */ clock_init(); + finishidentcpu(); /* Final stage of CPU initialization */ + setidt(IDT_UD, &IDTVEC(ill), SDT_SYS386TGT, SEL_KPL, + GSEL(GCODE_SEL, SEL_KPL)); + setidt(IDT_GP, &IDTVEC(prot), SDT_SYS386TGT, SEL_KPL, + GSEL(GCODE_SEL, SEL_KPL)); + initializecpu(); /* Initialize CPU registers */ + initializecpucache(); + + /* pointer to selector slot for %fs/%gs */ + PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd); + + dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 = + dblfault_tss.tss_esp2 = (int)&dblfault_stack[sizeof(dblfault_stack)]; + dblfault_tss.tss_ss = dblfault_tss.tss_ss0 = dblfault_tss.tss_ss1 = + dblfault_tss.tss_ss2 = GSEL(GDATA_SEL, SEL_KPL); +#if defined(PAE) || defined(PAE_TABLES) + dblfault_tss.tss_cr3 = (int)IdlePDPT; +#else + dblfault_tss.tss_cr3 = (int)IdlePTD; +#endif + dblfault_tss.tss_eip = (int)dblfault_handler; + dblfault_tss.tss_eflags = PSL_KERNEL; + dblfault_tss.tss_ds = dblfault_tss.tss_es = + dblfault_tss.tss_gs = GSEL(GDATA_SEL, SEL_KPL); + dblfault_tss.tss_fs = GSEL(GPRIV_SEL, SEL_KPL); + dblfault_tss.tss_cs = GSEL(GCODE_SEL, SEL_KPL); + dblfault_tss.tss_ldt = GSEL(GLDT_SEL, SEL_KPL); + + vm86_initialize(); + getmemsize(first); + init_param2(physmem); + + /* now running on new page tables, configured,and u/iom is accessible */ + /* * Initialize the console before we print anything out. */ @@ -2652,40 +2685,6 @@ init386(first) kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); #endif - finishidentcpu(); /* Final stage of CPU initialization */ - setidt(IDT_UD, &IDTVEC(ill), SDT_SYS386TGT, SEL_KPL, - GSEL(GCODE_SEL, SEL_KPL)); - setidt(IDT_GP, &IDTVEC(prot), SDT_SYS386TGT, SEL_KPL, - GSEL(GCODE_SEL, SEL_KPL)); - initializecpu(); /* Initialize CPU registers */ - initializecpucache(); - - /* pointer to selector slot for %fs/%gs */ - PCPU_SET(fsgs_gdt, &gdt[GUFS_SEL].sd); - - dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 = - dblfault_tss.tss_esp2 = (int)&dblfault_stack[sizeof(dblfault_stack)]; - dblfault_tss.tss_ss = dblfault_tss.tss_ss0 = dblfault_tss.tss_ss1 = - dblfault_tss.tss_ss2 = GSEL(GDATA_SEL, SEL_KPL); -#if defined(PAE) || defined(PAE_TABLES) - dblfault_tss.tss_cr3 = (int)IdlePDPT; -#else - dblfault_tss.tss_cr3 = (int)IdlePTD; -#endif - dblfault_tss.tss_eip = (int)dblfault_handler; - dblfault_tss.tss_eflags = PSL_KERNEL; - dblfault_tss.tss_ds = dblfault_tss.tss_es = - dblfault_tss.tss_gs = GSEL(GDATA_SEL, SEL_KPL); - dblfault_tss.tss_fs = GSEL(GPRIV_SEL, SEL_KPL); - dblfault_tss.tss_cs = GSEL(GCODE_SEL, SEL_KPL); - dblfault_tss.tss_ldt = GSEL(GLDT_SEL, SEL_KPL); - - vm86_initialize(); - getmemsize(first); - init_param2(physmem); - - /* now running on new page tables, configured,and u/iom is accessible */ - msgbufinit(msgbufp, msgbufsize); #ifdef DEV_NPX npxinit(true); @@ -2778,8 +2777,6 @@ smap_sysctl_handler(SYSCTL_HANDLER_ARGS) kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf32 kernel"); - if (kmdp == NULL) - return (0); smapbase = (struct bios_smap *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_SMAP); if (smapbase == NULL) diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 083aa4a..9b8b9c9 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -247,6 +247,8 @@ init_secondary(void) pc->pc_prvspace = pc; pc->pc_curthread = 0; + intel_fix_cpuid(); + gdt_segs[GPRIV_SEL].ssd_base = (int) pc; gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss; @@ -289,7 +291,7 @@ init_secondary(void) CHECK_WRITE(0x39, 6); /* Spin until the BSP releases the AP's. */ - while (!aps_ready) + while (atomic_load_acq_int(&aps_ready) == 0) ia32_pause(); /* BSP may have changed PTD while we were waiting */ @@ -346,7 +348,7 @@ start_all_aps(void) /* allocate and set up a boot stack data page */ bootstacks[cpu] = - (char *)kmem_malloc(kernel_arena, KSTACK_PAGES * PAGE_SIZE, + (char *)kmem_malloc(kernel_arena, kstack_pages * PAGE_SIZE, M_WAITOK | M_ZERO); dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, M_WAITOK | M_ZERO); @@ -358,7 +360,8 @@ start_all_aps(void) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 4; + bootSTK = (char *)bootstacks[cpu] + kstack_pages * + PAGE_SIZE - 4; bootAP = cpu; /* attempt to start the Application Processor */ @@ -501,7 +504,7 @@ smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) mtx_lock_spin(&smp_ipi_mtx); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; - atomic_store_rel_int(&smp_tlb_wait, 0); + smp_tlb_wait = 0; ipi_all_but_self(vector); while (smp_tlb_wait < ncpu) ia32_pause(); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index a642d10..9b8b2ff 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -118,11 +118,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vmmeter.h> #include <sys/sched.h> #include <sys/sysctl.h> -#ifdef SMP #include <sys/smp.h> -#else -#include <sys/cpuset.h> -#endif #include <vm/vm.h> #include <vm/vm_param.h> @@ -232,6 +228,18 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, #define PAT_INDEX_SIZE 8 static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ +/* + * pmap_mapdev support pre initialization (i.e. console) + */ +#define PMAP_PREINIT_MAPPING_COUNT 8 +static struct pmap_preinit_mapping { + vm_paddr_t pa; + vm_offset_t va; + vm_size_t sz; + int mode; +} pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT]; +static int pmap_initialized; + static struct rwlock_padalign pvh_global_lock; /* @@ -512,6 +520,22 @@ pmap_bootstrap(vm_paddr_t firstaddr) pmap_set_pg(); } +static void +pmap_init_qpages(void) +{ + struct pcpu *pc; + int i; + + CPU_FOREACH(i) { + pc = pcpu_find(i); + pc->pc_qmap_addr = kva_alloc(PAGE_SIZE); + if (pc->pc_qmap_addr == 0) + panic("pmap_init_qpages: unable to allocate KVA"); + } +} + +SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_qpages, NULL); + /* * Setup the PAT MSR. */ @@ -732,6 +756,7 @@ pmap_ptelist_init(vm_offset_t *head, void *base, int npages) void pmap_init(void) { + struct pmap_preinit_mapping *ppim; vm_page_t mpte; vm_size_t s; int i, pv_npg; @@ -815,6 +840,17 @@ pmap_init(void) UMA_ZONE_VM | UMA_ZONE_NOFREE); uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf); #endif + + pmap_initialized = 1; + if (!bootverbose) + return; + for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { + ppim = pmap_preinit_mapping + i; + if (ppim->va == 0) + continue; + printf("PPIM %u: PA=%#jx, VA=%#x, size=%#x, mode=%#x\n", i, + (uintmax_t)ppim->pa, ppim->va, ppim->sz, ppim->mode); + } } @@ -5059,8 +5095,10 @@ pmap_pde_attr(pd_entry_t *pde, int cache_bits) void * pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) { + struct pmap_preinit_mapping *ppim; vm_offset_t va, offset; vm_size_t tmpsize; + int i; offset = pa & PAGE_MASK; size = round_page(offset + size); @@ -5068,11 +5106,36 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) if (pa < KERNLOAD && pa + size <= KERNLOAD) va = KERNBASE + pa; - else + else if (!pmap_initialized) { + va = 0; + for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { + ppim = pmap_preinit_mapping + i; + if (ppim->va == 0) { + ppim->pa = pa; + ppim->sz = size; + ppim->mode = mode; + ppim->va = virtual_avail; + virtual_avail += size; + va = ppim->va; + break; + } + } + if (va == 0) + panic("%s: too many preinit mappings", __func__); + } else { + /* + * If we have a preinit mapping, re-use it. + */ + for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { + ppim = pmap_preinit_mapping + i; + if (ppim->pa == pa && ppim->sz == size && + ppim->mode == mode) + return ((void *)(ppim->va + offset)); + } va = kva_alloc(size); - if (!va) - panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); - + if (va == 0) + panic("%s: Couldn't allocate KVA", __func__); + } for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); pmap_invalidate_range(kernel_pmap, va, va + tmpsize); @@ -5097,14 +5160,31 @@ pmap_mapbios(vm_paddr_t pa, vm_size_t size) void pmap_unmapdev(vm_offset_t va, vm_size_t size) { - vm_offset_t base, offset; + struct pmap_preinit_mapping *ppim; + vm_offset_t offset; + int i; if (va >= KERNBASE && va + size <= KERNBASE + KERNLOAD) return; - base = trunc_page(va); offset = va & PAGE_MASK; size = round_page(offset + size); - kva_free(base, size); + va = trunc_page(va); + for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { + ppim = pmap_preinit_mapping + i; + if (ppim->va == va && ppim->sz == size) { + if (pmap_initialized) + return; + ppim->pa = 0; + ppim->va = 0; + ppim->sz = 0; + ppim->mode = 0; + if (va + size == virtual_avail) + virtual_avail = va; + return; + } + } + if (pmap_initialized) + kva_free(va, size); } /* @@ -5400,6 +5480,39 @@ pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset; } +vm_offset_t +pmap_quick_enter_page(vm_page_t m) +{ + vm_offset_t qaddr; + pt_entry_t *pte; + + critical_enter(); + qaddr = PCPU_GET(qmap_addr); + pte = vtopte(qaddr); + + KASSERT(*pte == 0, ("pmap_quick_enter_page: PTE busy")); + *pte = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M | + pmap_cache_bits(pmap_page_get_memattr(m), 0); + invlpg(qaddr); + + return (qaddr); +} + +void +pmap_quick_remove_page(vm_offset_t addr) +{ + vm_offset_t qaddr; + pt_entry_t *pte; + + qaddr = PCPU_GET(qmap_addr); + pte = vtopte(qaddr); + + KASSERT(*pte != 0, ("pmap_quick_remove_page: PTE not in use")); + KASSERT(addr == qaddr, ("pmap_quick_remove_page: invalid address")); + + *pte = 0; + critical_exit(); +} #if defined(PMAP_DEBUG) pmap_pid_dump(int pid) @@ -5461,51 +5574,3 @@ pmap_pid_dump(int pid) return (npte); } #endif - -#if defined(DEBUG) - -static void pads(pmap_t pm); -void pmap_pvdump(vm_paddr_t pa); - -/* print address space of pmap*/ -static void -pads(pmap_t pm) -{ - int i, j; - vm_paddr_t va; - pt_entry_t *ptep; - - if (pm == kernel_pmap) - return; - for (i = 0; i < NPDEPTD; i++) - if (pm->pm_pdir[i]) - for (j = 0; j < NPTEPG; j++) { - va = (i << PDRSHIFT) + (j << PAGE_SHIFT); - if (pm == kernel_pmap && va < KERNBASE) - continue; - if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) - continue; - ptep = pmap_pte(pm, va); - if (pmap_pte_v(ptep)) - printf("%x:%x ", va, *ptep); - }; - -} - -void -pmap_pvdump(vm_paddr_t pa) -{ - pv_entry_t pv; - pmap_t pmap; - vm_page_t m; - - printf("pa %x", pa); - m = PHYS_TO_VM_PAGE(pa); - TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { - pmap = PV_PMAP(pv); - printf(" -> pmap %p, va %x", (void *)pmap, pv->pv_va); - pads(pmap); - } - printf(" "); -} -#endif diff --git a/sys/i386/i386/stack_machdep.c b/sys/i386/i386/stack_machdep.c deleted file mode 100644 index 8dbb9ab..0000000 --- a/sys/i386/i386/stack_machdep.c +++ /dev/null @@ -1,87 +0,0 @@ -/*- - * Copyright (c) 2005 Antoine Brodin - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - -#include <sys/param.h> -#include <sys/systm.h> -#include <sys/proc.h> -#include <sys/stack.h> - -#include <machine/pcb.h> -#include <machine/stack.h> - -#include <vm/vm.h> -#include <vm/vm_param.h> -#include <vm/pmap.h> - -static void -stack_capture(struct thread *td, struct stack *st, register_t ebp) -{ - struct i386_frame *frame; - vm_offset_t callpc; - - stack_zero(st); - frame = (struct i386_frame *)ebp; - while (1) { - if (!INKERNEL(frame)) - break; - callpc = frame->f_retaddr; - if (!INKERNEL(callpc)) - break; - if (stack_put(st, callpc) == -1) - break; - if (frame->f_frame <= frame || - (vm_offset_t)frame->f_frame >= td->td_kstack + - td->td_kstack_pages * PAGE_SIZE) - break; - frame = frame->f_frame; - } -} - -void -stack_save_td(struct stack *st, struct thread *td) -{ - register_t ebp; - - if (TD_IS_SWAPPED(td)) - panic("stack_save_td: swapped"); - if (TD_IS_RUNNING(td)) - panic("stack_save_td: running"); - - ebp = td->td_pcb->pcb_ebp; - stack_capture(td, st, ebp); -} - -void -stack_save(struct stack *st) -{ - register_t ebp; - - __asm __volatile("movl %%ebp,%0" : "=r" (ebp)); - stack_capture(curthread, st, ebp); -} diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c index 0928b72..dc367a6 100644 --- a/sys/i386/i386/sys_machdep.c +++ b/sys/i386/i386/sys_machdep.c @@ -275,7 +275,7 @@ i386_extend_pcb(struct thread *td) ext = (struct pcb_ext *)kmem_malloc(kernel_arena, ctob(IOPAGES+1), M_WAITOK | M_ZERO); /* -16 is so we can convert a trapframe into vm86trapframe inplace */ - ext->ext_tss.tss_esp0 = td->td_kstack + ctob(KSTACK_PAGES) - + ext->ext_tss.tss_esp0 = td->td_kstack + ctob(td->td_kstack_pages) - sizeof(struct pcb) - 16; ext->ext_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); /* diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 1e417bf..40f7204 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include "opt_isa.h" #include "opt_kdb.h" #include "opt_npx.h" +#include "opt_stack.h" #include "opt_trap.h" #include <sys/param.h> @@ -94,6 +95,7 @@ PMC_SOFT_DEFINE( , , page_fault, write); #ifdef SMP #include <machine/smp.h> #endif +#include <machine/stack.h> #include <machine/tss.h> #include <machine/vm86.h> @@ -219,18 +221,25 @@ trap(struct trapframe *frame) goto out; } -#ifdef HWPMC_HOOKS - /* - * CPU PMCs interrupt using an NMI so we check for that first. - * If the HWPMC module is active, 'pmc_hook' will point to - * the function to be called. A return value of '1' from the - * hook means that the NMI was handled by it and that we can - * return immediately. - */ - if (type == T_NMI && pmc_intr && - (*pmc_intr)(PCPU_GET(cpuid), frame)) - goto out; + if (type == T_NMI) { +#ifdef HWPMC_HOOKS + /* + * CPU PMCs interrupt using an NMI so we check for that first. + * If the HWPMC module is active, 'pmc_hook' will point to + * the function to be called. A non-zero return value from the + * hook means that the NMI was consumed by it and that we can + * return immediately. + */ + if (pmc_intr != NULL && + (*pmc_intr)(PCPU_GET(cpuid), frame) != 0) + goto out; +#endif + +#ifdef STACK + if (stack_nmi_handler(frame) != 0) + goto out; #endif + } if (type == T_MCHK) { mca_intr(); @@ -782,7 +791,6 @@ trap_pfault(frame, usermode, eva) vm_offset_t eva; { vm_offset_t va; - struct vmspace *vm; vm_map_t map; int rv = 0; vm_prot_t ftype; @@ -852,14 +860,7 @@ trap_pfault(frame, usermode, eva) map = kernel_map; } else { - /* - * This is a fault on non-kernel virtual memory. If either - * p or p->p_vmspace is NULL, then the fault is fatal. - */ - if (p == NULL || (vm = p->p_vmspace) == NULL) - goto nogo; - - map = &vm->vm_map; + map = &p->p_vmspace->vm_map; /* * When accessing a user-space address, kernel must be @@ -888,28 +889,8 @@ trap_pfault(frame, usermode, eva) else ftype = VM_PROT_READ; - if (map != kernel_map) { - /* - * Keep swapout from messing with us during this - * critical time. - */ - PROC_LOCK(p); - ++p->p_lock; - PROC_UNLOCK(p); - - /* Fault in the user page: */ - rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL); - - PROC_LOCK(p); - --p->p_lock; - PROC_UNLOCK(p); - } else { - /* - * Don't have to worry about process locking or stacks in the - * kernel. - */ - rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL); - } + /* Fault in the page. */ + rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL); if (rv == KERN_SUCCESS) { #ifdef HWPMC_HOOKS if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index f67b279..62e99aa 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -111,8 +111,8 @@ _Static_assert(OFFSETOF_CURTHREAD == offsetof(struct pcpu, pc_curthread), "OFFSETOF_CURTHREAD does not correspond with offset of pc_curthread."); _Static_assert(OFFSETOF_CURPCB == offsetof(struct pcpu, pc_curpcb), "OFFSETOF_CURPCB does not correspond with offset of pc_curpcb."); -_Static_assert(OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf), - "OFFSETOF_MONINORBUF does not correspond with offset of pc_monitorbuf."); +_Static_assert(__OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf), + "__OFFSETOF_MONINORBUF does not correspond with offset of pc_monitorbuf."); static void cpu_reset_real(void); #ifdef SMP diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index adc86ca..3242d76 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -37,9 +37,31 @@ #include <machine/specialreg.h> #endif -#define mb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") -#define wmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") -#define rmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") +#ifndef __OFFSETOF_MONITORBUF +/* + * __OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). + * + * The open-coded number is used instead of the symbolic expression to + * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. + * An assertion in i386/vm_machdep.c ensures that the value is correct. + */ +#define __OFFSETOF_MONITORBUF 0x180 + +static __inline void +__mbk(void) +{ + + __asm __volatile("lock; addl $0,%%fs:%0" + : "+m" (*(u_int *)__OFFSETOF_MONITORBUF) : : "memory", "cc"); +} + +static __inline void +__mbu(void) +{ + + __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); +} +#endif /* * Various simple operations on memory, each of which is atomic in the @@ -233,71 +255,36 @@ atomic_testandset_int(volatile u_int *p, u_int v) * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct - * addresses, so for atomic_load_acq we introduce a Store/Load barrier - * before the load in SMP kernels. We use "lock addl $0,mem", as - * recommended by the AMD Software Optimization Guide, and not mfence. - * In the kernel, we use a private per-cpu cache line as the target - * for the locked addition, to avoid introducing false data - * dependencies. In userspace, a word at the top of the stack is - * utilized. + * addresses, so we need Store/Load barrier for sequentially + * consistent fences in SMP kernels. We use "lock addl $0,mem" for a + * Store/Load barrier, as recommended by the AMD Software Optimization + * Guide, and not mfence. In the kernel, we use a private per-cpu + * cache line for "mem", to avoid introducing false data + * dependencies. In user space, we use the word at the top of the + * stack. * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ -#if defined(_KERNEL) - -/* - * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). - * - * The open-coded number is used instead of the symbolic expression to - * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. - * An assertion in i386/vm_machdep.c ensures that the value is correct. - */ -#define OFFSETOF_MONITORBUF 0x180 +#if defined(_KERNEL) #if defined(SMP) -static __inline void -__storeload_barrier(void) -{ - - __asm __volatile("lock; addl $0,%%fs:%0" - : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); -} +#define __storeload_barrier() __mbk() #else /* _KERNEL && UP */ -static __inline void -__storeload_barrier(void) -{ - - __compiler_membar(); -} +#define __storeload_barrier() __compiler_membar() #endif /* SMP */ #else /* !_KERNEL */ -static __inline void -__storeload_barrier(void) -{ - - __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc"); -} +#define __storeload_barrier() __mbu() #endif /* _KERNEL*/ -/* - * C11-standard acq/rel semantics only apply when the variable in the - * call is the same for acq as it is for rel. However, our previous - * (x86) implementations provided much stronger ordering than required - * (essentially what is called seq_cst order in C11). This - * implementation provides the historical strong ordering since some - * callers depend on it. - */ - #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ - __storeload_barrier(); \ res = *p; \ __compiler_membar(); \ return (res); \ @@ -786,4 +773,14 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* !WANT_FUNCTIONS */ +#if defined(_KERNEL) +#define mb() __mbk() +#define wmb() __mbk() +#define rmb() __mbk() +#else +#define mb() __mbu() +#define wmb() __mbu() +#define rmb() __mbu() +#endif + #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/i386/include/in_cksum.h b/sys/i386/include/in_cksum.h index 34d85be..8816f3c 100644 --- a/sys/i386/include/in_cksum.h +++ b/sys/i386/include/in_cksum.h @@ -10,7 +10,7 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the University nor the names of its contributors + * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * diff --git a/sys/i386/include/md_var.h b/sys/i386/include/md_var.h index e610240..13655ee 100644 --- a/sys/i386/include/md_var.h +++ b/sys/i386/include/md_var.h @@ -118,6 +118,7 @@ void fillw(int /*u_short*/ pat, void *base, size_t cnt); void fill_based_sd(struct segment_descriptor *sdp, uint32_t base); void initializecpu(void); void initializecpucache(void); +bool intel_fix_cpuid(void); void i686_pagezero(void *addr); void sse2_pagezero(void *addr); void init_AMD_Elan_sc520(void); diff --git a/sys/i386/include/param.h b/sys/i386/include/param.h index b3fd85f..f7968f8 100644 --- a/sys/i386/include/param.h +++ b/sys/i386/include/param.h @@ -114,6 +114,11 @@ #define KSTACK_PAGES 2 /* Includes pcb! */ #endif #define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */ +#if KSTACK_PAGES < 4 +#define TD0_KSTACK_PAGES 4 +#else +#define TD0_KSTACK_PAGES KSTACK_PAGES +#endif /* * Ceiling on amount of swblock kva space, can be changed via diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h index 231f80f..cfd8d03 100644 --- a/sys/i386/include/pcpu.h +++ b/sys/i386/include/pcpu.h @@ -58,7 +58,8 @@ int pc_private_tss; /* Flag indicating private tss*/\ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ u_int pc_vcpu_id; /* Xen vCPU ID */ \ - char __pad[233] + vm_offset_t pc_qmap_addr; /* KVA for temporary mappings */\ + char __pad[229] #ifdef _KERNEL diff --git a/sys/i386/include/privatespace.h b/sys/i386/include/privatespace.h deleted file mode 100644 index 5eb54c2..0000000 --- a/sys/i386/include/privatespace.h +++ /dev/null @@ -1,49 +0,0 @@ -/*- - * Copyright (c) Peter Wemm - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _MACHINE_PRIVATESPACE_H_ -#define _MACHINE_PRIVATESPACE_H_ - -/* - * This is the upper (0xff800000) address space layout that is per-cpu. - * It is setup in locore.s and pmap.c for the BSP and in mp_machdep.c for - * each AP. This is only applicable to the x86 SMP kernel. - */ -struct privatespace { - /* page 0 - data page */ - struct pcpu pcpu; - char __filler0[PAGE_SIZE - sizeof(struct pcpu)]; - - /* page 1 - idle stack (KSTACK_PAGES pages) */ - char idlekstack[KSTACK_PAGES * PAGE_SIZE]; - /* page 1+KSTACK_PAGES... */ -}; - -extern struct privatespace SMP_prvspace[]; - -#endif /* ! _MACHINE_PRIVATESPACE_H_ */ diff --git a/sys/i386/include/stack.h b/sys/i386/include/stack.h index f63fc4b..091ae33 100644 --- a/sys/i386/include/stack.h +++ b/sys/i386/include/stack.h @@ -1,42 +1,6 @@ -/*- - * Mach Operating System - * Copyright (c) 1991,1990 Carnegie Mellon University - * All Rights Reserved. - * - * Permission to use, copy, modify and distribute this software and its - * documentation is hereby granted, provided that both the copyright - * notice and this permission notice appear in all copies of the - * software, derivative works or modified versions, and any portions - * thereof, and that both notices appear in supporting documentation. - * - * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS - * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR - * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * - * Carnegie Mellon requests users of this software to return to - * - * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU - * School of Computer Science - * Carnegie Mellon University - * Pittsburgh PA 15213-3890 - * - * any improvements or extensions that they make and grant Carnegie the - * rights to redistribute these changes. - * - * $FreeBSD$ - */ - -#ifndef _MACHINE_STACK_H_ -#define _MACHINE_STACK_H_ - /* - * Stack trace. + * This file is in the public domain. */ +/* $FreeBSD$ */ -struct i386_frame { - struct i386_frame *f_frame; - int f_retaddr; - int f_arg0; -}; - -#endif /* !_MACHINE_STACK_H_ */ +#include <x86/stack.h> diff --git a/sys/i386/include/xen/xenfunc.h b/sys/i386/include/xen/xenfunc.h deleted file mode 100644 index f48b1f1..0000000 --- a/sys/i386/include/xen/xenfunc.h +++ /dev/null @@ -1,81 +0,0 @@ -/*- - * Copyright (c) 2004, 2005 Kip Macy - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _XEN_XENFUNC_H_ -#define _XEN_XENFUNC_H_ - -#include <xen/xen-os.h> -#include <xen/hypervisor.h> - -#include <vm/pmap.h> - -#include <machine/segments.h> - -#include <sys/pcpu.h> -#define BKPT __asm__("int3"); -#define XPQ_CALL_DEPTH 5 -#define XPQ_CALL_COUNT 2 -#define PG_PRIV PG_AVAIL3 -typedef struct { - unsigned long pt_ref; - unsigned long pt_eip[XPQ_CALL_COUNT][XPQ_CALL_DEPTH]; -} pteinfo_t; - -extern pteinfo_t *pteinfo_list; -#ifdef XENDEBUG_LOW -#define __PRINTK(x) printk x -#else -#define __PRINTK(x) -#endif - -char *xen_setbootenv(char *cmd_line); - -int xen_boothowto(char *envp); - -void _xen_machphys_update(vm_paddr_t, vm_paddr_t, char *file, int line); - -#ifdef INVARIANTS -#define xen_machphys_update(a, b) _xen_machphys_update((a), (b), __FILE__, __LINE__) -#else -#define xen_machphys_update(a, b) _xen_machphys_update((a), (b), NULL, 0) -#endif - -void xen_update_descriptor(union descriptor *, union descriptor *); - -extern struct mtx balloon_lock; -#if 0 -#define balloon_lock(__flags) mtx_lock_irqsave(&balloon_lock, __flags) -#define balloon_unlock(__flags) mtx_unlock_irqrestore(&balloon_lock, __flags) -#else -#define balloon_lock(__flags) __flags = 1 -#define balloon_unlock(__flags) __flags = 0 -#endif - - - -#endif /* _XEN_XENFUNC_H_ */ diff --git a/sys/i386/include/xen/xenvar.h b/sys/i386/include/xen/xenvar.h deleted file mode 100644 index 484c279..0000000 --- a/sys/i386/include/xen/xenvar.h +++ /dev/null @@ -1,36 +0,0 @@ -/*- - * Copyright (c) 2008 Kip Macy - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef XENVAR_H_ -#define XENVAR_H_ - -#include <xen/features.h> - -#define vtomach(va) pmap_kextract((vm_offset_t) (va)) - -#endif |
