diff options
Diffstat (limited to 'sys/x86/xen/hvm.c')
| -rw-r--r-- | sys/x86/xen/hvm.c | 447 |
1 files changed, 439 insertions, 8 deletions
diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index c689b47..0404fe9 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008 Citrix Systems, Inc. + * Copyright (c) 2008, 2013 Citrix Systems, Inc. * Copyright (c) 2012 Spectra Logic Corporation * All rights reserved. * @@ -33,9 +33,19 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/proc.h> +#include <sys/smp.h> +#include <sys/systm.h> + +#include <vm/vm.h> +#include <vm/pmap.h> #include <dev/pci/pcivar.h> + #include <machine/cpufunc.h> +#include <machine/cpu.h> +#include <machine/smp.h> + +#include <x86/apicreg.h> #include <xen/xen-os.h> #include <xen/features.h> @@ -44,30 +54,450 @@ __FBSDID("$FreeBSD$"); #include <xen/hvm.h> #include <xen/xen_intr.h> -#include <vm/vm.h> -#include <vm/pmap.h> - #include <xen/interface/hvm/params.h> #include <xen/interface/vcpu.h> +/*--------------------------- Forward Declarations ---------------------------*/ +static driver_filter_t xen_smp_rendezvous_action; +static driver_filter_t xen_invltlb; +static driver_filter_t xen_invlpg; +static driver_filter_t xen_invlrng; +static driver_filter_t xen_invlcache; +#ifdef __i386__ +static driver_filter_t xen_lazypmap; +#endif +static driver_filter_t xen_ipi_bitmap_handler; +static driver_filter_t xen_cpustop_handler; +static driver_filter_t xen_cpususpend_handler; +static driver_filter_t xen_cpustophard_handler; + +/*---------------------------- Extern Declarations ---------------------------*/ +/* Variables used by mp_machdep to perform the MMU related IPIs */ +extern volatile int smp_tlb_wait; +extern vm_offset_t smp_tlb_addr2; +#ifdef __i386__ +extern vm_offset_t smp_tlb_addr1; +#else +extern struct invpcid_descr smp_tlb_invpcid; +extern uint64_t pcid_cr3; +extern int invpcid_works; +extern int pmap_pcid_enabled; +extern pmap_t smp_tlb_pmap; +#endif + +#ifdef __i386__ +extern void pmap_lazyfix_action(void); +#endif + +/*---------------------------------- Macros ----------------------------------*/ +#define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) + +/*-------------------------------- Local Types -------------------------------*/ +struct xen_ipi_handler +{ + driver_filter_t *filter; + const char *description; +}; + +/*-------------------------------- Global Data -------------------------------*/ +enum xen_domain_type xen_domain_type = XEN_NATIVE; + static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support"); -DPCPU_DEFINE(struct vcpu_info, vcpu_local_info); -DPCPU_DEFINE(struct vcpu_info *, vcpu_info); +static struct xen_ipi_handler xen_ipis[] = +{ + [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, + [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, + [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, + [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, + [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, +#ifdef __i386__ + [IPI_TO_IDX(IPI_LAZYPMAP)] = { xen_lazypmap, "lp" }, +#endif + [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, + [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, + [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, + [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, +}; -/*-------------------------------- Global Data -------------------------------*/ /** * If non-zero, the hypervisor has been configured to use a direct * IDT event callback for interrupt injection. */ int xen_vector_callback_enabled; +/*------------------------------- Per-CPU Data -------------------------------*/ +DPCPU_DEFINE(struct vcpu_info, vcpu_local_info); +DPCPU_DEFINE(struct vcpu_info *, vcpu_info); +DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); + /*------------------ Hypervisor Access Shared Memory Regions -----------------*/ /** Hypercall table accessed via HYPERVISOR_*_op() methods. */ char *hypercall_stubs; shared_info_t *HYPERVISOR_shared_info; -enum xen_domain_type xen_domain_type = XEN_NATIVE; +/*---------------------------- XEN PV IPI Handlers ---------------------------*/ +/* + * This are C clones of the ASM functions found in apic_vector.s + */ +static int +xen_ipi_bitmap_handler(void *arg) +{ + struct trapframe *frame; + + frame = arg; + ipi_bitmap_handler(*frame); + return (FILTER_HANDLED); +} + +static int +xen_smp_rendezvous_action(void *arg) +{ +#ifdef COUNT_IPIS + int cpu; + + cpu = PCPU_GET(cpuid); + (*ipi_rendezvous_counts[cpu])++; +#endif /* COUNT_IPIS */ + + smp_rendezvous_action(); + return (FILTER_HANDLED); +} + +static int +xen_invltlb(void *arg) +{ +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + int cpu; + + cpu = PCPU_GET(cpuid); +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[cpu]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[cpu])++; +#endif /* COUNT_IPIS */ +#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ + + invltlb(); + atomic_add_int(&smp_tlb_wait, 1); + return (FILTER_HANDLED); +} + +#ifdef __amd64__ +static int +xen_invltlb_pcid(void *arg) +{ + uint64_t cr3; +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + int cpu; + + cpu = PCPU_GET(cpuid); +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[cpu]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[cpu])++; +#endif /* COUNT_IPIS */ +#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ + + cr3 = rcr3(); + if (smp_tlb_invpcid.pcid != (uint64_t)-1 && + smp_tlb_invpcid.pcid != 0) { + + if (invpcid_works) { + invpcid(&smp_tlb_invpcid, INVPCID_CTX); + } else { + /* Otherwise reload %cr3 twice. */ + if (cr3 != pcid_cr3) { + load_cr3(pcid_cr3); + cr3 |= CR3_PCID_SAVE; + } + load_cr3(cr3); + } + } else { + invltlb_globpcid(); + } + if (smp_tlb_pmap != NULL) + CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); + + atomic_add_int(&smp_tlb_wait, 1); + return (FILTER_HANDLED); +} +#endif + +static int +xen_invlpg(void *arg) +{ +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + int cpu; + + cpu = PCPU_GET(cpuid); +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[cpu]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[cpu])++; +#endif /* COUNT_IPIS */ +#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ + +#ifdef __i386__ + invlpg(smp_tlb_addr1); +#else + invlpg(smp_tlb_invpcid.addr); +#endif + atomic_add_int(&smp_tlb_wait, 1); + return (FILTER_HANDLED); +} + +#ifdef __amd64__ +static int +xen_invlpg_pcid(void *arg) +{ +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + int cpu; + + cpu = PCPU_GET(cpuid); +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[cpu]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[cpu])++; +#endif /* COUNT_IPIS */ +#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ + + if (invpcid_works) { + invpcid(&smp_tlb_invpcid, INVPCID_ADDR); + } else if (smp_tlb_invpcid.pcid == 0) { + invlpg(smp_tlb_invpcid.addr); + } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { + invltlb_globpcid(); + } else { + uint64_t cr3; + + /* + * PCID supported, but INVPCID is not. + * Temporarily switch to the target address + * space and do INVLPG. + */ + cr3 = rcr3(); + if (cr3 != pcid_cr3) + load_cr3(pcid_cr3 | CR3_PCID_SAVE); + invlpg(smp_tlb_invpcid.addr); + load_cr3(cr3 | CR3_PCID_SAVE); + } + + atomic_add_int(&smp_tlb_wait, 1); + return (FILTER_HANDLED); +} +#endif + +static inline void +invlpg_range(vm_offset_t start, vm_offset_t end) +{ + do { + invlpg(start); + start += PAGE_SIZE; + } while (start < end); +} + +static int +xen_invlrng(void *arg) +{ + vm_offset_t addr; +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + int cpu; + + cpu = PCPU_GET(cpuid); +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[cpu]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlrng_counts[cpu])++; +#endif /* COUNT_IPIS */ +#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ + +#ifdef __i386__ + addr = smp_tlb_addr1; + invlpg_range(addr, smp_tlb_addr2); +#else + addr = smp_tlb_invpcid.addr; + if (pmap_pcid_enabled) { + if (invpcid_works) { + struct invpcid_descr d; + + d = smp_tlb_invpcid; + do { + invpcid(&d, INVPCID_ADDR); + d.addr += PAGE_SIZE; + } while (d.addr < smp_tlb_addr2); + } else if (smp_tlb_invpcid.pcid == 0) { + /* + * kernel pmap - use invlpg to invalidate + * global mapping. + */ + invlpg_range(addr, smp_tlb_addr2); + } else if (smp_tlb_invpcid.pcid != (uint64_t)-1) { + invltlb_globpcid(); + if (smp_tlb_pmap != NULL) { + CPU_CLR_ATOMIC(PCPU_GET(cpuid), + &smp_tlb_pmap->pm_save); + } + } else { + uint64_t cr3; + + cr3 = rcr3(); + if (cr3 != pcid_cr3) + load_cr3(pcid_cr3 | CR3_PCID_SAVE); + invlpg_range(addr, smp_tlb_addr2); + load_cr3(cr3 | CR3_PCID_SAVE); + } + } else { + invlpg_range(addr, smp_tlb_addr2); + } +#endif + + atomic_add_int(&smp_tlb_wait, 1); + return (FILTER_HANDLED); +} + +static int +xen_invlcache(void *arg) +{ +#ifdef COUNT_IPIS + int cpu = PCPU_GET(cpuid); + + cpu = PCPU_GET(cpuid); + (*ipi_invlcache_counts[cpu])++; +#endif /* COUNT_IPIS */ + + wbinvd(); + atomic_add_int(&smp_tlb_wait, 1); + return (FILTER_HANDLED); +} + +#ifdef __i386__ +static int +xen_lazypmap(void *arg) +{ + + pmap_lazyfix_action(); + return (FILTER_HANDLED); +} +#endif + +static int +xen_cpustop_handler(void *arg) +{ + + cpustop_handler(); + return (FILTER_HANDLED); +} + +static int +xen_cpususpend_handler(void *arg) +{ + + cpususpend_handler(); + return (FILTER_HANDLED); +} + +static int +xen_cpustophard_handler(void *arg) +{ + + ipi_nmi_handler(); + return (FILTER_HANDLED); +} + +/* Xen PV IPI sender */ +static void +xen_ipi_vectored(u_int vector, int dest) +{ + xen_intr_handle_t *ipi_handle; + int ipi_idx, to_cpu, self; + + ipi_idx = IPI_TO_IDX(vector); + if (ipi_idx > nitems(xen_ipis)) + panic("IPI out of range"); + + switch(dest) { + case APIC_IPI_DEST_SELF: + ipi_handle = DPCPU_GET(ipi_handle); + xen_intr_signal(ipi_handle[ipi_idx]); + break; + case APIC_IPI_DEST_ALL: + CPU_FOREACH(to_cpu) { + ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); + xen_intr_signal(ipi_handle[ipi_idx]); + } + break; + case APIC_IPI_DEST_OTHERS: + self = PCPU_GET(cpuid); + CPU_FOREACH(to_cpu) { + if (to_cpu != self) { + ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); + xen_intr_signal(ipi_handle[ipi_idx]); + } + } + break; + default: + to_cpu = apic_cpuid(dest); + ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); + xen_intr_signal(ipi_handle[ipi_idx]); + break; + } +} + +static void +xen_cpu_ipi_init(int cpu) +{ + xen_intr_handle_t *ipi_handle; + const struct xen_ipi_handler *ipi; + device_t dev; + int idx, rc; + + ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); + dev = pcpu_find(cpu)->pc_device; + KASSERT((dev != NULL), ("NULL pcpu device_t")); + + for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { + + if (ipi->filter == NULL) { + ipi_handle[idx] = NULL; + continue; + } + + rc = xen_intr_alloc_and_bind_ipi(dev, cpu, ipi->filter, + INTR_TYPE_TTY, &ipi_handle[idx]); + if (rc != 0) + panic("Unable to allocate a XEN IPI port"); + xen_intr_describe(ipi_handle[idx], "%s", ipi->description); + } +} + +static void +xen_init_ipis(void) +{ + int i; + + if (!xen_hvm_domain() || !xen_vector_callback_enabled) + return; + +#ifdef __amd64__ + if (pmap_pcid_enabled) { + xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = xen_invltlb_pcid; + xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = xen_invlpg_pcid; + } +#endif + CPU_FOREACH(i) + xen_cpu_ipi_init(i); + + /* Set the xen pv ipi ops to replace the native ones */ + cpu_ops.ipi_vectored = xen_ipi_vectored; +} + +/*---------------------- XEN Hypervisor Probe and Setup ----------------------*/ static uint32_t xen_hvm_cpuid_base(void) { @@ -253,4 +683,5 @@ void xen_hvm_init_cpu(void) } SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_init, NULL); +SYSINIT(xen_init_ipis, SI_SUB_SMP, SI_ORDER_FIRST, xen_init_ipis, NULL); SYSINIT(xen_hvm_init_cpu, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_init_cpu, NULL); |
