From b3aff6ccbb1d25e506b60ccd9c559013903f3464 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Wed, 3 Feb 2016 16:56:51 +0000 Subject: KVM: arm/arm64: Fix reference to uninitialised VGIC Commit 4b4b4512da2a ("arm/arm64: KVM: Rework the arch timer to use level-triggered semantics") brought the virtual architected timer closer to the VGIC. There is one occasion were we don't properly check for the VGIC actually having been initialized before, but instead go on to check the active state of some IRQ number. If userland hasn't instantiated a virtual GIC, we end up with a kernel NULL pointer dereference: ========= Unable to handle kernel NULL pointer dereference at virtual address 00000000 pgd = ffffffc9745c5000 [00000000] *pgd=00000009f631e003, *pud=00000009f631e003, *pmd=0000000000000000 Internal error: Oops: 96000006 [#2] PREEMPT SMP Modules linked in: CPU: 0 PID: 2144 Comm: kvm_simplest-ar Tainted: G D 4.5.0-rc2+ #1300 Hardware name: ARM Juno development board (r1) (DT) task: ffffffc976da8000 ti: ffffffc976e28000 task.ti: ffffffc976e28000 PC is at vgic_bitmap_get_irq_val+0x78/0x90 LR is at kvm_vgic_map_is_active+0xac/0xc8 pc : [] lr : [] pstate: 20000145 .... ========= Fix this by bailing out early of kvm_timer_flush_hwstate() if we don't have a VGIC at all. Reported-by: Cosmin Gorgovan Acked-by: Marc Zyngier Signed-off-by: Andre Przywara Signed-off-by: Marc Zyngier Cc: # 4.4.x --- virt/kvm/arm/arch_timer.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index 69bca18..ea60646 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -143,7 +143,7 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level) * Check if there was a change in the timer state (should we raise or lower * the line level to the GIC). */ -static void kvm_timer_update_state(struct kvm_vcpu *vcpu) +static int kvm_timer_update_state(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; @@ -154,10 +154,12 @@ static void kvm_timer_update_state(struct kvm_vcpu *vcpu) * until we call this function from kvm_timer_flush_hwstate. */ if (!vgic_initialized(vcpu->kvm)) - return; + return -ENODEV; if (kvm_timer_should_fire(vcpu) != timer->irq.level) kvm_timer_update_irq(vcpu, !timer->irq.level); + + return 0; } /* @@ -218,7 +220,8 @@ void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu) bool phys_active; int ret; - kvm_timer_update_state(vcpu); + if (kvm_timer_update_state(vcpu)) + return; /* * If we enter the guest with the virtual input level to the VGIC -- cgit v1.1 From 236cf17c2502007a9d2dda3c39fb0d9a6bd03cc2 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 16 Feb 2016 14:47:31 +0000 Subject: KVM: arm/arm64: vgic: Ensure bitmaps are long enough When we allocate bitmaps in vgic_vcpu_init_maps, we divide the number of bits we need by 8 to figure out how many bytes to allocate. However, bitmap elements are always accessed as unsigned longs, and if we didn't happen to allocate a size such that size % sizeof(unsigned long) == 0, bitmap accesses may go past the end of the allocation. When using KASAN (which does byte-granular access checks), this results in a continuous stream of BUGs whenever these bitmaps are accessed: ============================================================================= BUG kmalloc-128 (Tainted: G B ): kasan: bad access detected ----------------------------------------------------------------------------- INFO: Allocated in vgic_init.part.25+0x55c/0x990 age=7493 cpu=3 pid=1730 INFO: Slab 0xffffffbde6d5da40 objects=16 used=15 fp=0xffffffc935769700 flags=0x4000000000000080 INFO: Object 0xffffffc935769500 @offset=1280 fp=0x (null) Bytes b4 ffffffc9357694f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769500: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769510: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769520: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769530: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769540: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769550: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Object ffffffc935769570: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Padding ffffffc9357695b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Padding ffffffc9357695c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Padding ffffffc9357695d0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Padding ffffffc9357695e0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ Padding ffffffc9357695f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ CPU: 3 PID: 1740 Comm: kvm-vcpu-0 Tainted: G B 4.4.0+ #17 Hardware name: ARM Juno development board (r1) (DT) Call trace: [] dump_backtrace+0x0/0x280 [] show_stack+0x14/0x20 [] dump_stack+0x100/0x188 [] print_trailer+0xfc/0x168 [] object_err+0x3c/0x50 [] kasan_report_error+0x244/0x558 [] __asan_report_load8_noabort+0x48/0x50 [] __bitmap_or+0xc0/0xc8 [] kvm_vgic_flush_hwstate+0x1bc/0x650 [] kvm_arch_vcpu_ioctl_run+0x2ec/0xa60 [] kvm_vcpu_ioctl+0x474/0xa68 [] do_vfs_ioctl+0x5b8/0xcb0 [] SyS_ioctl+0x8c/0xa0 [] el0_svc_naked+0x24/0x28 Memory state around the buggy address: ffffffc935769400: 00 00 fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffffffc935769480: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc >ffffffc935769500: 04 fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ^ ffffffc935769580: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffffffc935769600: 00 00 00 00 fc fc fc fc fc fc fc fc fc fc fc fc ================================================================== Fix the issue by always allocating a multiple of sizeof(unsigned long), as we do elsewhere in the vgic code. Fixes: c1bfb577a ("arm/arm64: KVM: vgic: switch to dynamic allocation") Cc: stable@vger.kernel.org Acked-by: Marc Zyngier Acked-by: Christoffer Dall Signed-off-by: Mark Rutland Signed-off-by: Marc Zyngier --- virt/kvm/arm/vgic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c index 043032c..00429b3 100644 --- a/virt/kvm/arm/vgic.c +++ b/virt/kvm/arm/vgic.c @@ -1875,8 +1875,8 @@ void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu) static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs) { struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - - int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8; + int nr_longs = BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS); + int sz = nr_longs * sizeof(unsigned long); vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL); vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL); vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL); -- cgit v1.1 From d7444794a02ff655eda87e3cc54e86b940e7736f Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Fri, 19 Feb 2016 13:11:46 +0100 Subject: KVM: async_pf: do not warn on page allocation failures In async_pf we try to allocate with NOWAIT to get an element quickly or fail. This code also handle failures gracefully. Lets silence potential page allocation failures under load. qemu-system-s39: page allocation failure: order:0,mode:0x2200000 [...] Call Trace: ([<00000000001146b8>] show_trace+0xf8/0x148) [<000000000011476a>] show_stack+0x62/0xe8 [<00000000004a36b8>] dump_stack+0x70/0x98 [<0000000000272c3a>] warn_alloc_failed+0xd2/0x148 [<000000000027709e>] __alloc_pages_nodemask+0x94e/0xb38 [<00000000002cd36a>] new_slab+0x382/0x400 [<00000000002cf7ac>] ___slab_alloc.constprop.30+0x2dc/0x378 [<00000000002d03d0>] kmem_cache_alloc+0x160/0x1d0 [<0000000000133db4>] kvm_setup_async_pf+0x6c/0x198 [<000000000013dee8>] kvm_arch_vcpu_ioctl_run+0xd48/0xd58 [<000000000012fcaa>] kvm_vcpu_ioctl+0x372/0x690 [<00000000002f66f6>] do_vfs_ioctl+0x3be/0x510 [<00000000002f68ec>] SyS_ioctl+0xa4/0xb8 [<0000000000781c5e>] system_call+0xd6/0x264 [<000003ffa24fa06a>] 0x3ffa24fa06a Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Dominik Dingel Signed-off-by: Paolo Bonzini --- virt/kvm/async_pf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'virt') diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 3531599..db2dd33 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -172,7 +172,7 @@ int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, unsigned long hva, * do alloc nowait since if we are going to sleep anyway we * may as well sleep faulting in page */ - work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT); + work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT | __GFP_NOWARN); if (!work) return 0; -- cgit v1.1 From 6d50d54cd80cb60d701e70fb0633250115606459 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 28 Jan 2016 14:24:39 +0000 Subject: arm64: KVM: Move vgic-v2 and timer save/restore to virt/kvm/arm/hyp We already have virt/kvm/arm/ containing timer and vgic stuff. Add yet another subdirectory to contain the hyp-specific files (timer and vgic again). Acked-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/hyp/timer-sr.c | 69 ++++++++++++++++++++++++++++++++++++ virt/kvm/arm/hyp/vgic-v2-sr.c | 82 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 virt/kvm/arm/hyp/timer-sr.c create mode 100644 virt/kvm/arm/hyp/vgic-v2-sr.c (limited to 'virt') diff --git a/virt/kvm/arm/hyp/timer-sr.c b/virt/kvm/arm/hyp/timer-sr.c new file mode 100644 index 0000000..ea00d69 --- /dev/null +++ b/virt/kvm/arm/hyp/timer-sr.c @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2012-2015 - ARM Ltd + * Author: Marc Zyngier + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#include + +/* vcpu is already in the HYP VA space */ +void __hyp_text __timer_save_state(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = kern_hyp_va(vcpu->kvm); + struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + u64 val; + + if (kvm->arch.timer.enabled) { + timer->cntv_ctl = read_sysreg_el0(cntv_ctl); + timer->cntv_cval = read_sysreg_el0(cntv_cval); + } + + /* Disable the virtual timer */ + write_sysreg_el0(0, cntv_ctl); + + /* Allow physical timer/counter access for the host */ + val = read_sysreg(cnthctl_el2); + val |= CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN; + write_sysreg(val, cnthctl_el2); + + /* Clear cntvoff for the host */ + write_sysreg(0, cntvoff_el2); +} + +void __hyp_text __timer_restore_state(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = kern_hyp_va(vcpu->kvm); + struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + u64 val; + + /* + * Disallow physical timer access for the guest + * Physical counter access is allowed + */ + val = read_sysreg(cnthctl_el2); + val &= ~CNTHCTL_EL1PCEN; + val |= CNTHCTL_EL1PCTEN; + write_sysreg(val, cnthctl_el2); + + if (kvm->arch.timer.enabled) { + write_sysreg(kvm->arch.timer.cntvoff, cntvoff_el2); + write_sysreg_el0(timer->cntv_cval, cntv_cval); + isb(); + write_sysreg_el0(timer->cntv_ctl, cntv_ctl); + } +} diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c new file mode 100644 index 0000000..9514a7d --- /dev/null +++ b/virt/kvm/arm/hyp/vgic-v2-sr.c @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2012-2015 - ARM Ltd + * Author: Marc Zyngier + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#include + +/* vcpu is already in the HYP VA space */ +void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = kern_hyp_va(vcpu->kvm); + struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; + struct vgic_dist *vgic = &kvm->arch.vgic; + void __iomem *base = kern_hyp_va(vgic->vctrl_base); + u32 eisr0, eisr1, elrsr0, elrsr1; + int i, nr_lr; + + if (!base) + return; + + nr_lr = vcpu->arch.vgic_cpu.nr_lr; + cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR); + cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR); + eisr0 = readl_relaxed(base + GICH_EISR0); + elrsr0 = readl_relaxed(base + GICH_ELRSR0); + if (unlikely(nr_lr > 32)) { + eisr1 = readl_relaxed(base + GICH_EISR1); + elrsr1 = readl_relaxed(base + GICH_ELRSR1); + } else { + eisr1 = elrsr1 = 0; + } +#ifdef CONFIG_CPU_BIG_ENDIAN + cpu_if->vgic_eisr = ((u64)eisr0 << 32) | eisr1; + cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) | elrsr1; +#else + cpu_if->vgic_eisr = ((u64)eisr1 << 32) | eisr0; + cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) | elrsr0; +#endif + cpu_if->vgic_apr = readl_relaxed(base + GICH_APR); + + writel_relaxed(0, base + GICH_HCR); + + for (i = 0; i < nr_lr; i++) + cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); +} + +/* vcpu is already in the HYP VA space */ +void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = kern_hyp_va(vcpu->kvm); + struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; + struct vgic_dist *vgic = &kvm->arch.vgic; + void __iomem *base = kern_hyp_va(vgic->vctrl_base); + int i, nr_lr; + + if (!base) + return; + + writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR); + writel_relaxed(cpu_if->vgic_vmcr, base + GICH_VMCR); + writel_relaxed(cpu_if->vgic_apr, base + GICH_APR); + + nr_lr = vcpu->arch.vgic_cpu.nr_lr; + for (i = 0; i < nr_lr; i++) + writel_relaxed(cpu_if->vgic_lr[i], base + GICH_LR0 + (i * 4)); +} -- cgit v1.1 From 051ff581ce70e822729e9474941f3c206cbf7436 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Tue, 8 Dec 2015 15:29:06 +0800 Subject: arm64: KVM: Add access handler for event counter register These kind of registers include PMEVCNTRn, PMCCNTR and PMXEVCNTR which is mapped to PMEVCNTRn. The access handler translates all aarch32 register offsets to aarch64 ones and uses vcpu_sys_reg() to access their values to avoid taking care of big endian. When reading these registers, return the sum of register value and the value perf event counts. Signed-off-by: Shannon Zhao Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 virt/kvm/arm/pmu.c (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c new file mode 100644 index 0000000..cd74e63 --- /dev/null +++ b/virt/kvm/arm/pmu.c @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2015 Linaro Ltd. + * Author: Shannon Zhao + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +/** + * kvm_pmu_get_counter_value - get PMU counter value + * @vcpu: The vcpu pointer + * @select_idx: The counter index + */ +u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx) +{ + u64 counter, reg, enabled, running; + struct kvm_pmu *pmu = &vcpu->arch.pmu; + struct kvm_pmc *pmc = &pmu->pmc[select_idx]; + + reg = (select_idx == ARMV8_PMU_CYCLE_IDX) + ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx; + counter = vcpu_sys_reg(vcpu, reg); + + /* The real counter value is equal to the value of counter register plus + * the value perf event counts. + */ + if (pmc->perf_event) + counter += perf_event_read_value(pmc->perf_event, &enabled, + &running); + + return counter & pmc->bitmask; +} + +/** + * kvm_pmu_set_counter_value - set PMU counter value + * @vcpu: The vcpu pointer + * @select_idx: The counter index + * @val: The counter value + */ +void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) +{ + u64 reg; + + reg = (select_idx == ARMV8_PMU_CYCLE_IDX) + ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx; + vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx); +} -- cgit v1.1 From 96b0eebcc6a14e3bdb9ff0e7176fbfc225bdde94 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Tue, 8 Sep 2015 12:26:13 +0800 Subject: arm64: KVM: Add access handler for PMCNTENSET and PMCNTENCLR register Since the reset value of PMCNTENSET and PMCNTENCLR is UNKNOWN, use reset_unknown for its reset handler. Add a handler to emulate writing PMCNTENSET or PMCNTENCLR register. When writing to PMCNTENSET, call perf_event_enable to enable the perf event. When writing to PMCNTENCLR, call perf_event_disable to disable the perf event. Signed-off-by: Shannon Zhao Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index cd74e63..f8dc174 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -61,3 +61,69 @@ void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx; vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx); } + +u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) +{ + u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT; + + val &= ARMV8_PMU_PMCR_N_MASK; + if (val == 0) + return BIT(ARMV8_PMU_CYCLE_IDX); + else + return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX); +} + +/** + * kvm_pmu_enable_counter - enable selected PMU counter + * @vcpu: The vcpu pointer + * @val: the value guest writes to PMCNTENSET register + * + * Call perf_event_enable to start counting the perf event + */ +void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val) +{ + int i; + struct kvm_pmu *pmu = &vcpu->arch.pmu; + struct kvm_pmc *pmc; + + if (!(vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val) + return; + + for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) { + if (!(val & BIT(i))) + continue; + + pmc = &pmu->pmc[i]; + if (pmc->perf_event) { + perf_event_enable(pmc->perf_event); + if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE) + kvm_debug("fail to enable perf event\n"); + } + } +} + +/** + * kvm_pmu_disable_counter - disable selected PMU counter + * @vcpu: The vcpu pointer + * @val: the value guest writes to PMCNTENCLR register + * + * Call perf_event_disable to stop counting the perf event + */ +void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val) +{ + int i; + struct kvm_pmu *pmu = &vcpu->arch.pmu; + struct kvm_pmc *pmc; + + if (!val) + return; + + for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) { + if (!(val & BIT(i))) + continue; + + pmc = &pmu->pmc[i]; + if (pmc->perf_event) + perf_event_disable(pmc->perf_event); + } +} -- cgit v1.1 From 7f7663587165fe1a81c3390358cb70eb7234706f Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Fri, 3 Jul 2015 14:27:25 +0800 Subject: arm64: KVM: PMU: Add perf event map and introduce perf event creating function When we use tools like perf on host, perf passes the event type and the id of this event type category to kernel, then kernel will map them to hardware event number and write this number to PMU PMEVTYPER_EL0 register. When getting the event number in KVM, directly use raw event type to create a perf_event for it. Signed-off-by: Shannon Zhao Reviewed-by: Marc Zyngier Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index f8dc174..591a11d 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -62,6 +62,27 @@ void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx); } +/** + * kvm_pmu_stop_counter - stop PMU counter + * @pmc: The PMU counter pointer + * + * If this counter has been configured to monitor some event, release it here. + */ +static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc) +{ + u64 counter, reg; + + if (pmc->perf_event) { + counter = kvm_pmu_get_counter_value(vcpu, pmc->idx); + reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX) + ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx; + vcpu_sys_reg(vcpu, reg) = counter; + perf_event_disable(pmc->perf_event); + perf_event_release_kernel(pmc->perf_event); + pmc->perf_event = NULL; + } +} + u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) { u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT; @@ -127,3 +148,56 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val) perf_event_disable(pmc->perf_event); } } + +static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx) +{ + return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) && + (vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx)); +} + +/** + * kvm_pmu_set_counter_event_type - set selected counter to monitor some event + * @vcpu: The vcpu pointer + * @data: The data guest writes to PMXEVTYPER_EL0 + * @select_idx: The number of selected counter + * + * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an + * event with given hardware event number. Here we call perf_event API to + * emulate this action and create a kernel perf event for it. + */ +void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, + u64 select_idx) +{ + struct kvm_pmu *pmu = &vcpu->arch.pmu; + struct kvm_pmc *pmc = &pmu->pmc[select_idx]; + struct perf_event *event; + struct perf_event_attr attr; + u64 eventsel, counter; + + kvm_pmu_stop_counter(vcpu, pmc); + eventsel = data & ARMV8_PMU_EVTYPE_EVENT; + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.type = PERF_TYPE_RAW; + attr.size = sizeof(attr); + attr.pinned = 1; + attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, select_idx); + attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0; + attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0; + attr.exclude_hv = 1; /* Don't count EL2 events */ + attr.exclude_host = 1; /* Don't count host events */ + attr.config = eventsel; + + counter = kvm_pmu_get_counter_value(vcpu, select_idx); + /* The initial sample period (overflow count) of an event. */ + attr.sample_period = (-counter) & pmc->bitmask; + + event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc); + if (IS_ERR(event)) { + pr_err_once("kvm: pmu event creation failed %ld\n", + PTR_ERR(event)); + return; + } + + pmc->perf_event = event; +} -- cgit v1.1 From 76d883c4e6401b98ea26d40c437ff62719a517ad Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Tue, 8 Sep 2015 15:03:26 +0800 Subject: arm64: KVM: Add access handler for PMOVSSET and PMOVSCLR register Since the reset value of PMOVSSET and PMOVSCLR is UNKNOWN, use reset_unknown for its reset handler. Add a handler to emulate writing PMOVSSET or PMOVSCLR register. When writing non-zero value to PMOVSSET, the counter and its interrupt is enabled, kick this vcpu to sync PMU interrupt. Signed-off-by: Shannon Zhao Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 591a11d..0232861 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -149,6 +149,37 @@ void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val) } } +static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) +{ + u64 reg = 0; + + if ((vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) + reg = vcpu_sys_reg(vcpu, PMOVSSET_EL0); + reg &= vcpu_sys_reg(vcpu, PMCNTENSET_EL0); + reg &= vcpu_sys_reg(vcpu, PMINTENSET_EL1); + reg &= kvm_pmu_valid_counter_mask(vcpu); + + return reg; +} + +/** + * kvm_pmu_overflow_set - set PMU overflow interrupt + * @vcpu: The vcpu pointer + * @val: the value guest writes to PMOVSSET register + */ +void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val) +{ + u64 reg; + + if (val == 0) + return; + + vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= val; + reg = kvm_pmu_overflow_status(vcpu); + if (reg != 0) + kvm_vcpu_kick(vcpu); +} + static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx) { return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) && -- cgit v1.1 From 7a0adc7064b88609e2917446af8789fac1d4fdd1 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Tue, 8 Sep 2015 15:49:39 +0800 Subject: arm64: KVM: Add access handler for PMSWINC register Add access handler which emulates writing and reading PMSWINC register and add support for creating software increment event. Signed-off-by: Shannon Zhao Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 0232861..9fc775e 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -180,6 +180,36 @@ void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val) kvm_vcpu_kick(vcpu); } +/** + * kvm_pmu_software_increment - do software increment + * @vcpu: The vcpu pointer + * @val: the value guest writes to PMSWINC register + */ +void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) +{ + int i; + u64 type, enable, reg; + + if (val == 0) + return; + + enable = vcpu_sys_reg(vcpu, PMCNTENSET_EL0); + for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) { + if (!(val & BIT(i))) + continue; + type = vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i) + & ARMV8_PMU_EVTYPE_EVENT; + if ((type == ARMV8_PMU_EVTYPE_EVENT_SW_INCR) + && (enable & BIT(i))) { + reg = vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1; + reg = lower_32_bits(reg); + vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg; + if (!reg) + kvm_pmu_overflow_set(vcpu, BIT(i)); + } + } +} + static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx) { return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) && @@ -208,6 +238,10 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, kvm_pmu_stop_counter(vcpu, pmc); eventsel = data & ARMV8_PMU_EVTYPE_EVENT; + /* Software increment event does't need to be backed by a perf event */ + if (eventsel == ARMV8_PMU_EVTYPE_EVENT_SW_INCR) + return; + memset(&attr, 0, sizeof(struct perf_event_attr)); attr.type = PERF_TYPE_RAW; attr.size = sizeof(attr); -- cgit v1.1 From 76993739cd6f5b42e881fe3332b9f8eb98cd6907 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Wed, 28 Oct 2015 12:10:30 +0800 Subject: arm64: KVM: Add helper to handle PMCR register bits According to ARMv8 spec, when writing 1 to PMCR.E, all counters are enabled by PMCNTENSET, while writing 0 to PMCR.E, all counters are disabled. When writing 1 to PMCR.P, reset all event counters, not including PMCCNTR, to zero. When writing 1 to PMCR.C, reset PMCCNTR to zero. Signed-off-by: Shannon Zhao Reviewed-by: Marc Zyngier Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 9fc775e..cda869c 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -210,6 +210,40 @@ void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) } } +/** + * kvm_pmu_handle_pmcr - handle PMCR register + * @vcpu: The vcpu pointer + * @val: the value guest writes to PMCR register + */ +void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) +{ + struct kvm_pmu *pmu = &vcpu->arch.pmu; + struct kvm_pmc *pmc; + u64 mask; + int i; + + mask = kvm_pmu_valid_counter_mask(vcpu); + if (val & ARMV8_PMU_PMCR_E) { + kvm_pmu_enable_counter(vcpu, + vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask); + } else { + kvm_pmu_disable_counter(vcpu, mask); + } + + if (val & ARMV8_PMU_PMCR_C) + kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0); + + if (val & ARMV8_PMU_PMCR_P) { + for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) + kvm_pmu_set_counter_value(vcpu, i, 0); + } + + if (val & ARMV8_PMU_PMCR_LC) { + pmc = &pmu->pmc[ARMV8_PMU_CYCLE_IDX]; + pmc->bitmask = 0xffffffffffffffffUL; + } +} + static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx) { return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) && -- cgit v1.1 From b02386eb7dac7555a208d81aef2a0e5c6f0f8085 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Fri, 26 Feb 2016 19:29:19 +0800 Subject: arm64: KVM: Add PMU overflow interrupt routing When calling perf_event_create_kernel_counter to create perf_event, assign a overflow handler. Then when the perf event overflows, set the corresponding bit of guest PMOVSSET register. If this counter is enabled and its interrupt is enabled as well, kick the vcpu to sync the interrupt. On VM entry, if there is counter overflowed and interrupt level is changed, inject the interrupt with corresponding level. On VM exit, sync the interrupt level as well if it has been changed. Signed-off-by: Shannon Zhao Reviewed-by: Marc Zyngier Reviewed-by: Andrew Jones Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index cda869c..74e858c 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -21,6 +21,7 @@ #include #include #include +#include /** * kvm_pmu_get_counter_value - get PMU counter value @@ -180,6 +181,71 @@ void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val) kvm_vcpu_kick(vcpu); } +static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) +{ + struct kvm_pmu *pmu = &vcpu->arch.pmu; + bool overflow; + + if (!kvm_arm_pmu_v3_ready(vcpu)) + return; + + overflow = !!kvm_pmu_overflow_status(vcpu); + if (pmu->irq_level != overflow) { + pmu->irq_level = overflow; + kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, + pmu->irq_num, overflow); + } +} + +/** + * kvm_pmu_flush_hwstate - flush pmu state to cpu + * @vcpu: The vcpu pointer + * + * Check if the PMU has overflowed while we were running in the host, and inject + * an interrupt if that was the case. + */ +void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) +{ + kvm_pmu_update_state(vcpu); +} + +/** + * kvm_pmu_sync_hwstate - sync pmu state from cpu + * @vcpu: The vcpu pointer + * + * Check if the PMU has overflowed while we were running in the guest, and + * inject an interrupt if that was the case. + */ +void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) +{ + kvm_pmu_update_state(vcpu); +} + +static inline struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc) +{ + struct kvm_pmu *pmu; + struct kvm_vcpu_arch *vcpu_arch; + + pmc -= pmc->idx; + pmu = container_of(pmc, struct kvm_pmu, pmc[0]); + vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu); + return container_of(vcpu_arch, struct kvm_vcpu, arch); +} + +/** + * When perf event overflows, call kvm_pmu_overflow_set to set overflow status. + */ +static void kvm_pmu_perf_overflow(struct perf_event *perf_event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct kvm_pmc *pmc = perf_event->overflow_handler_context; + struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); + int idx = pmc->idx; + + kvm_pmu_overflow_set(vcpu, BIT(idx)); +} + /** * kvm_pmu_software_increment - do software increment * @vcpu: The vcpu pointer @@ -291,7 +357,8 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, /* The initial sample period (overflow count) of an event. */ attr.sample_period = (-counter) & pmc->bitmask; - event = perf_event_create_kernel_counter(&attr, -1, current, NULL, pmc); + event = perf_event_create_kernel_counter(&attr, -1, current, + kvm_pmu_perf_overflow, pmc); if (IS_ERR(event)) { pr_err_once("kvm: pmu event creation failed %ld\n", PTR_ERR(event)); -- cgit v1.1 From 2aa36e9840d71710f06b3c29634f044fde8bcbe5 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Fri, 11 Sep 2015 11:30:22 +0800 Subject: arm64: KVM: Reset PMU state when resetting vcpu When resetting vcpu, it needs to reset the PMU state to initial status. Signed-off-by: Shannon Zhao Reviewed-by: Marc Zyngier Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 74e858c..1dbbc2c 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -84,6 +84,23 @@ static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc) } } +/** + * kvm_pmu_vcpu_reset - reset pmu state for cpu + * @vcpu: The vcpu pointer + * + */ +void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) +{ + int i; + struct kvm_pmu *pmu = &vcpu->arch.pmu; + + for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) { + kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]); + pmu->pmc[i].idx = i; + pmu->pmc[i].bitmask = 0xffffffffUL; + } +} + u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) { u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT; -- cgit v1.1 From 5f0a714a2b63c25ffba5d832773f3ca4f0d02e21 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Fri, 11 Sep 2015 15:18:05 +0800 Subject: arm64: KVM: Free perf event of PMU when destroying vcpu When KVM frees VCPU, it needs to free the perf_event of PMU. Signed-off-by: Shannon Zhao Reviewed-by: Marc Zyngier Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 1dbbc2c..9b83857 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -101,6 +101,27 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) } } +/** + * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu + * @vcpu: The vcpu pointer + * + */ +void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + int i; + struct kvm_pmu *pmu = &vcpu->arch.pmu; + + for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) { + struct kvm_pmc *pmc = &pmu->pmc[i]; + + if (pmc->perf_event) { + perf_event_disable(pmc->perf_event); + perf_event_release_kernel(pmc->perf_event); + pmc->perf_event = NULL; + } + } +} + u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) { u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT; -- cgit v1.1 From 808e738142e7086ef793ebf9797099c392894e65 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Mon, 11 Jan 2016 22:46:15 +0800 Subject: arm64: KVM: Add a new feature bit for PMUv3 To support guest PMUv3, use one bit of the VCPU INIT feature array. Initialize the PMU when initialzing the vcpu with that bit and PMU overflow interrupt set. Signed-off-by: Shannon Zhao Acked-by: Peter Maydell Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 9b83857..6e28f4f 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -405,3 +405,13 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, pmc->perf_event = event; } + +bool kvm_arm_support_pmu_v3(void) +{ + /* + * Check if HW_PERF_EVENTS are supported by checking the number of + * hardware performance counters. This could ensure the presence of + * a physical PMU and CONFIG_PERF_EVENT is selected. + */ + return (perf_num_counters() > 0); +} -- cgit v1.1 From bb0c70bcca6ba3c84afc2da7426f3b923bbe6825 Mon Sep 17 00:00:00 2001 From: Shannon Zhao Date: Mon, 11 Jan 2016 21:35:32 +0800 Subject: arm64: KVM: Add a new vcpu device control group for PMUv3 To configure the virtual PMUv3 overflow interrupt number, we use the vcpu kvm_device ioctl, encapsulating the KVM_ARM_VCPU_PMU_V3_IRQ attribute within the KVM_ARM_VCPU_PMU_V3_CTRL group. After configuring the PMUv3, call the vcpu ioctl with attribute KVM_ARM_VCPU_PMU_V3_INIT to initialize the PMUv3. Signed-off-by: Shannon Zhao Acked-by: Peter Maydell Reviewed-by: Andrew Jones Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/pmu.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/pmu.c b/virt/kvm/arm/pmu.c index 6e28f4f..b5754c6 100644 --- a/virt/kvm/arm/pmu.c +++ b/virt/kvm/arm/pmu.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -415,3 +416,114 @@ bool kvm_arm_support_pmu_v3(void) */ return (perf_num_counters() > 0); } + +static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) +{ + if (!kvm_arm_support_pmu_v3()) + return -ENODEV; + + if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features) || + !kvm_arm_pmu_irq_initialized(vcpu)) + return -ENXIO; + + if (kvm_arm_pmu_v3_ready(vcpu)) + return -EBUSY; + + kvm_pmu_vcpu_reset(vcpu); + vcpu->arch.pmu.ready = true; + + return 0; +} + +static bool irq_is_valid(struct kvm *kvm, int irq, bool is_ppi) +{ + int i; + struct kvm_vcpu *vcpu; + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (!kvm_arm_pmu_irq_initialized(vcpu)) + continue; + + if (is_ppi) { + if (vcpu->arch.pmu.irq_num != irq) + return false; + } else { + if (vcpu->arch.pmu.irq_num == irq) + return false; + } + } + + return true; +} + + +int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case KVM_ARM_VCPU_PMU_V3_IRQ: { + int __user *uaddr = (int __user *)(long)attr->addr; + int irq; + + if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features)) + return -ENODEV; + + if (get_user(irq, uaddr)) + return -EFAULT; + + /* + * The PMU overflow interrupt could be a PPI or SPI, but for one + * VM the interrupt type must be same for each vcpu. As a PPI, + * the interrupt number is the same for all vcpus, while as an + * SPI it must be a separate number per vcpu. + */ + if (irq < VGIC_NR_SGIS || irq >= vcpu->kvm->arch.vgic.nr_irqs || + !irq_is_valid(vcpu->kvm, irq, irq < VGIC_NR_PRIVATE_IRQS)) + return -EINVAL; + + if (kvm_arm_pmu_irq_initialized(vcpu)) + return -EBUSY; + + kvm_debug("Set kvm ARM PMU irq: %d\n", irq); + vcpu->arch.pmu.irq_num = irq; + return 0; + } + case KVM_ARM_VCPU_PMU_V3_INIT: + return kvm_arm_pmu_v3_init(vcpu); + } + + return -ENXIO; +} + +int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case KVM_ARM_VCPU_PMU_V3_IRQ: { + int __user *uaddr = (int __user *)(long)attr->addr; + int irq; + + if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features)) + return -ENODEV; + + if (!kvm_arm_pmu_irq_initialized(vcpu)) + return -ENXIO; + + irq = vcpu->arch.pmu.irq_num; + return put_user(irq, uaddr); + } + } + + return -ENXIO; +} + +int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) +{ + switch (attr->attr) { + case KVM_ARM_VCPU_PMU_V3_IRQ: + case KVM_ARM_VCPU_PMU_V3_INIT: + if (kvm_arm_support_pmu_v3() && + test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features)) + return 0; + } + + return -ENXIO; +} -- cgit v1.1 From 9b4a3004439d5be680faf41f4267968ca11bb9f6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 29 Jan 2016 19:04:48 +0000 Subject: KVM: arm/arm64: timer: Add active state caching Programming the active state in the (re)distributor can be an expensive operation so it makes some sense to try and reduce the number of accesses as much as possible. So far, we program the active state on each VM entry, but there is some opportunity to do less. An obvious solution is to cache the active state in memory, and only program it in the HW when conditions change. But because the HW can also change things under our feet (the active state can transition from 1 to 0 when the guest does an EOI), some precautions have to be taken, which amount to only caching an "inactive" state, and always programing it otherwise. With this in place, we observe a reduction of around 700 cycles on a 2GHz GICv2 platform for a NULL hypercall. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/arch_timer.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index ea60646..a9ad4fe 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -34,6 +34,11 @@ static struct timecounter *timecounter; static struct workqueue_struct *wqueue; static unsigned int host_vtimer_irq; +void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) +{ + vcpu->arch.timer_cpu.active_cleared_last = false; +} + static cycle_t kvm_phys_timer_read(void) { return timecounter->cc->read(timecounter->cc); @@ -130,6 +135,7 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level) BUG_ON(!vgic_initialized(vcpu->kvm)); + timer->active_cleared_last = false; timer->irq.level = new_level; trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->map->virt_irq, timer->irq.level); @@ -245,10 +251,35 @@ void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu) else phys_active = false; + /* + * We want to avoid hitting the (re)distributor as much as + * possible, as this is a potentially expensive MMIO access + * (not to mention locks in the irq layer), and a solution for + * this is to cache the "active" state in memory. + * + * Things to consider: we cannot cache an "active set" state, + * because the HW can change this behind our back (it becomes + * "clear" in the HW). We must then restrict the caching to + * the "clear" state. + * + * The cache is invalidated on: + * - vcpu put, indicating that the HW cannot be trusted to be + * in a sane state on the next vcpu load, + * - any change in the interrupt state + * + * Usage conditions: + * - cached value is "active clear" + * - value to be programmed is "active clear" + */ + if (timer->active_cleared_last && !phys_active) + return; + ret = irq_set_irqchip_state(timer->map->irq, IRQCHIP_STATE_ACTIVE, phys_active); WARN_ON(ret); + + timer->active_cleared_last = !phys_active; } /** -- cgit v1.1 From 59f00ff9afc028053fa9281407627e95008ebd5c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 2 Feb 2016 19:35:34 +0000 Subject: KVM: arm/arm64: vgic-v2: Avoid accessing GICH registers GICv2 registers are *slow*. As in "terrifyingly slow". Which is bad. But we're equaly bad, as we make a point in accessing them even if we don't have any interrupt in flight. A good solution is to first find out if we have anything useful to write into the GIC, and if we don't, to simply not do it. This involves tracking which LRs actually have something valid there. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/hyp/vgic-v2-sr.c | 72 ++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 22 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c index 9514a7d..aa0fdb8 100644 --- a/virt/kvm/arm/hyp/vgic-v2-sr.c +++ b/virt/kvm/arm/hyp/vgic-v2-sr.c @@ -36,28 +36,41 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) nr_lr = vcpu->arch.vgic_cpu.nr_lr; cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR); - cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR); - eisr0 = readl_relaxed(base + GICH_EISR0); - elrsr0 = readl_relaxed(base + GICH_ELRSR0); - if (unlikely(nr_lr > 32)) { - eisr1 = readl_relaxed(base + GICH_EISR1); - elrsr1 = readl_relaxed(base + GICH_ELRSR1); - } else { - eisr1 = elrsr1 = 0; - } + + if (vcpu->arch.vgic_cpu.live_lrs) { + eisr0 = readl_relaxed(base + GICH_EISR0); + elrsr0 = readl_relaxed(base + GICH_ELRSR0); + cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR); + cpu_if->vgic_apr = readl_relaxed(base + GICH_APR); + + if (unlikely(nr_lr > 32)) { + eisr1 = readl_relaxed(base + GICH_EISR1); + elrsr1 = readl_relaxed(base + GICH_ELRSR1); + } else { + eisr1 = elrsr1 = 0; + } + #ifdef CONFIG_CPU_BIG_ENDIAN - cpu_if->vgic_eisr = ((u64)eisr0 << 32) | eisr1; - cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) | elrsr1; + cpu_if->vgic_eisr = ((u64)eisr0 << 32) | eisr1; + cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) | elrsr1; #else - cpu_if->vgic_eisr = ((u64)eisr1 << 32) | eisr0; - cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) | elrsr0; + cpu_if->vgic_eisr = ((u64)eisr1 << 32) | eisr0; + cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) | elrsr0; #endif - cpu_if->vgic_apr = readl_relaxed(base + GICH_APR); - writel_relaxed(0, base + GICH_HCR); + for (i = 0; i < nr_lr; i++) + if (vcpu->arch.vgic_cpu.live_lrs & (1UL << i)) + cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); - for (i = 0; i < nr_lr; i++) - cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); + writel_relaxed(0, base + GICH_HCR); + + vcpu->arch.vgic_cpu.live_lrs = 0; + } else { + cpu_if->vgic_eisr = 0; + cpu_if->vgic_elrsr = ~0UL; + cpu_if->vgic_misr = 0; + cpu_if->vgic_apr = 0; + } } /* vcpu is already in the HYP VA space */ @@ -68,15 +81,30 @@ void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu) struct vgic_dist *vgic = &kvm->arch.vgic; void __iomem *base = kern_hyp_va(vgic->vctrl_base); int i, nr_lr; + u64 live_lrs = 0; if (!base) return; - writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR); - writel_relaxed(cpu_if->vgic_vmcr, base + GICH_VMCR); - writel_relaxed(cpu_if->vgic_apr, base + GICH_APR); - nr_lr = vcpu->arch.vgic_cpu.nr_lr; + for (i = 0; i < nr_lr; i++) - writel_relaxed(cpu_if->vgic_lr[i], base + GICH_LR0 + (i * 4)); + if (cpu_if->vgic_lr[i] & GICH_LR_STATE) + live_lrs |= 1UL << i; + + if (live_lrs) { + writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR); + writel_relaxed(cpu_if->vgic_apr, base + GICH_APR); + for (i = 0; i < nr_lr; i++) { + u32 val = 0; + + if (live_lrs & (1UL << i)) + val = cpu_if->vgic_lr[i]; + + writel_relaxed(val, base + GICH_LR0 + (i * 4)); + } + } + + writel_relaxed(cpu_if->vgic_vmcr, base + GICH_VMCR); + vcpu->arch.vgic_cpu.live_lrs = live_lrs; } -- cgit v1.1 From c813bb17f2b7166adb1c740ffc0d957b46271fa1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 9 Feb 2016 17:01:33 +0000 Subject: KVM: arm/arm64: vgic-v2: Save maintenance interrupt state only if required Next on our list of useless accesses is the maintenance interrupt status registers (GICH_MISR, GICH_EISR{0,1}). It is pointless to save them if we haven't asked for a maintenance interrupt the first place, which can only happen for two reasons: - Underflow: GICH_HCR_UIE will be set, - EOI: GICH_LR_EOI will be set. These conditions can be checked on the in-memory copies of the regs. Should any of these two condition be valid, we must read GICH_MISR. We can then check for GICH_MISR_EOI, and only when set read GICH_EISR*. This means that in most case, we don't have to save them at all. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/hyp/vgic-v2-sr.c | 54 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c index aa0fdb8..0dd83d5 100644 --- a/virt/kvm/arm/hyp/vgic-v2-sr.c +++ b/virt/kvm/arm/hyp/vgic-v2-sr.c @@ -21,6 +21,49 @@ #include +static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu, + void __iomem *base) +{ + struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; + int nr_lr = vcpu->arch.vgic_cpu.nr_lr; + u32 eisr0, eisr1; + int i; + bool expect_mi; + + expect_mi = !!(cpu_if->vgic_hcr & GICH_HCR_UIE); + + for (i = 0; i < nr_lr; i++) { + if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i))) + continue; + + expect_mi |= (!(cpu_if->vgic_lr[i] & GICH_LR_HW) && + (cpu_if->vgic_lr[i] & GICH_LR_EOI)); + } + + if (expect_mi) { + cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR); + + if (cpu_if->vgic_misr & GICH_MISR_EOI) { + eisr0 = readl_relaxed(base + GICH_EISR0); + if (unlikely(nr_lr > 32)) + eisr1 = readl_relaxed(base + GICH_EISR1); + else + eisr1 = 0; + } else { + eisr0 = eisr1 = 0; + } + } else { + cpu_if->vgic_misr = 0; + eisr0 = eisr1 = 0; + } + +#ifdef CONFIG_CPU_BIG_ENDIAN + cpu_if->vgic_eisr = ((u64)eisr0 << 32) | eisr1; +#else + cpu_if->vgic_eisr = ((u64)eisr1 << 32) | eisr0; +#endif +} + /* vcpu is already in the HYP VA space */ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) { @@ -28,7 +71,7 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; struct vgic_dist *vgic = &kvm->arch.vgic; void __iomem *base = kern_hyp_va(vgic->vctrl_base); - u32 eisr0, eisr1, elrsr0, elrsr1; + u32 elrsr0, elrsr1; int i, nr_lr; if (!base) @@ -38,26 +81,23 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR); if (vcpu->arch.vgic_cpu.live_lrs) { - eisr0 = readl_relaxed(base + GICH_EISR0); elrsr0 = readl_relaxed(base + GICH_ELRSR0); - cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR); cpu_if->vgic_apr = readl_relaxed(base + GICH_APR); if (unlikely(nr_lr > 32)) { - eisr1 = readl_relaxed(base + GICH_EISR1); elrsr1 = readl_relaxed(base + GICH_ELRSR1); } else { - eisr1 = elrsr1 = 0; + elrsr1 = 0; } #ifdef CONFIG_CPU_BIG_ENDIAN - cpu_if->vgic_eisr = ((u64)eisr0 << 32) | eisr1; cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) | elrsr1; #else - cpu_if->vgic_eisr = ((u64)eisr1 << 32) | eisr0; cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) | elrsr0; #endif + save_maint_int_state(vcpu, base); + for (i = 0; i < nr_lr; i++) if (vcpu->arch.vgic_cpu.live_lrs & (1UL << i)) cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); -- cgit v1.1 From 2a1044f8b7bfd99296774f5f90abf8f96013a997 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 9 Feb 2016 17:07:18 +0000 Subject: KVM: arm/arm64: vgic-v2: Move GICH_ELRSR saving to its own function In order to make the saving path slightly more readable and prepare for some more optimizations, let's move the GICH_ELRSR saving to its own function. No functional change. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/hyp/vgic-v2-sr.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c index 0dd83d5..c576228 100644 --- a/virt/kvm/arm/hyp/vgic-v2-sr.c +++ b/virt/kvm/arm/hyp/vgic-v2-sr.c @@ -64,6 +64,25 @@ static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu, #endif } +static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base) +{ + struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; + int nr_lr = vcpu->arch.vgic_cpu.nr_lr; + u32 elrsr0, elrsr1; + + elrsr0 = readl_relaxed(base + GICH_ELRSR0); + if (unlikely(nr_lr > 32)) + elrsr1 = readl_relaxed(base + GICH_ELRSR1); + else + elrsr1 = 0; + +#ifdef CONFIG_CPU_BIG_ENDIAN + cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) | elrsr1; +#else + cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) | elrsr0; +#endif +} + /* vcpu is already in the HYP VA space */ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) { @@ -71,7 +90,6 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; struct vgic_dist *vgic = &kvm->arch.vgic; void __iomem *base = kern_hyp_va(vgic->vctrl_base); - u32 elrsr0, elrsr1; int i, nr_lr; if (!base) @@ -81,22 +99,10 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR); if (vcpu->arch.vgic_cpu.live_lrs) { - elrsr0 = readl_relaxed(base + GICH_ELRSR0); - cpu_if->vgic_apr = readl_relaxed(base + GICH_APR); - - if (unlikely(nr_lr > 32)) { - elrsr1 = readl_relaxed(base + GICH_ELRSR1); - } else { - elrsr1 = 0; - } - -#ifdef CONFIG_CPU_BIG_ENDIAN - cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) | elrsr1; -#else - cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) | elrsr0; -#endif + cpu_if->vgic_apr = readl_relaxed(base + GICH_APR); save_maint_int_state(vcpu, base); + save_elrsr(vcpu, base); for (i = 0; i < nr_lr; i++) if (vcpu->arch.vgic_cpu.live_lrs & (1UL << i)) -- cgit v1.1 From f8cfbce1bb85984d2601a18e8f21dacc1b79dce3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 9 Feb 2016 17:09:49 +0000 Subject: KVM: arm/arm64: vgic-v2: Do not save an LR known to be empty On exit, any empty LR will be signaled in GICH_ELRSR*. Which means that we do not have to save it, and we can just clear its state in the in-memory copy. Take this opportunity to move the LR saving code into its own function. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/hyp/vgic-v2-sr.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c index c576228..6d4dd78 100644 --- a/virt/kvm/arm/hyp/vgic-v2-sr.c +++ b/virt/kvm/arm/hyp/vgic-v2-sr.c @@ -83,6 +83,25 @@ static void __hyp_text save_elrsr(struct kvm_vcpu *vcpu, void __iomem *base) #endif } +static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base) +{ + struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; + int nr_lr = vcpu->arch.vgic_cpu.nr_lr; + int i; + + for (i = 0; i < nr_lr; i++) { + if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i))) + continue; + + if (cpu_if->vgic_elrsr & (1UL << i)) { + cpu_if->vgic_lr[i] &= ~GICH_LR_STATE; + continue; + } + + cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); + } +} + /* vcpu is already in the HYP VA space */ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) { @@ -90,12 +109,10 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; struct vgic_dist *vgic = &kvm->arch.vgic; void __iomem *base = kern_hyp_va(vgic->vctrl_base); - int i, nr_lr; if (!base) return; - nr_lr = vcpu->arch.vgic_cpu.nr_lr; cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR); if (vcpu->arch.vgic_cpu.live_lrs) { @@ -103,10 +120,7 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) save_maint_int_state(vcpu, base); save_elrsr(vcpu, base); - - for (i = 0; i < nr_lr; i++) - if (vcpu->arch.vgic_cpu.live_lrs & (1UL << i)) - cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); + save_lrs(vcpu, base); writel_relaxed(0, base + GICH_HCR); -- cgit v1.1 From d6400d77463d5c3dd386c27d07a236a07daaf33e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 3 Mar 2016 15:43:58 +0000 Subject: KVM: arm/arm64: vgic-v2: Reset LRs at boot time In order to let make the GICv2 code more lazy in the way it accesses the LRs, it is necessary to start with a clean slate. Let's reset the LRs on each CPU when the vgic is probed. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/vgic-v2.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/vgic-v2.c b/virt/kvm/arm/vgic-v2.c index ff02f08..67ec334 100644 --- a/virt/kvm/arm/vgic-v2.c +++ b/virt/kvm/arm/vgic-v2.c @@ -176,6 +176,15 @@ static const struct vgic_ops vgic_v2_ops = { static struct vgic_params vgic_v2_params; +static void vgic_cpu_init_lrs(void *params) +{ + struct vgic_params *vgic = params; + int i; + + for (i = 0; i < vgic->nr_lr; i++) + writel_relaxed(0, vgic->vctrl_base + GICH_LR0 + (i * 4)); +} + /** * vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT * @node: pointer to the DT node @@ -257,6 +266,9 @@ int vgic_v2_probe(struct device_node *vgic_node, vgic->type = VGIC_V2; vgic->max_gic_vcpus = VGIC_V2_MAX_CPUS; + + on_each_cpu(vgic_cpu_init_lrs, vgic, 1); + *ops = &vgic_v2_ops; *params = vgic; goto out; -- cgit v1.1 From cc1daf0b82f12040065bb1a77dd7945b9ef821f8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 9 Feb 2016 17:36:09 +0000 Subject: KVM: arm/arm64: vgic-v2: Only wipe LRs on vcpu exit So far, we're always writing all possible LRs, setting the empty ones with a zero value. This is obvious doing a lot of work for nothing, and we're better off clearing those we've actually dirtied on the exit path (it is very rare to inject more than one interrupt at a time anyway). Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/hyp/vgic-v2-sr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/hyp/vgic-v2-sr.c b/virt/kvm/arm/hyp/vgic-v2-sr.c index 6d4dd78..674bdf8 100644 --- a/virt/kvm/arm/hyp/vgic-v2-sr.c +++ b/virt/kvm/arm/hyp/vgic-v2-sr.c @@ -99,6 +99,7 @@ static void __hyp_text save_lrs(struct kvm_vcpu *vcpu, void __iomem *base) } cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4)); + writel_relaxed(0, base + GICH_LR0 + (i * 4)); } } @@ -156,12 +157,11 @@ void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu) writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR); writel_relaxed(cpu_if->vgic_apr, base + GICH_APR); for (i = 0; i < nr_lr; i++) { - u32 val = 0; - - if (live_lrs & (1UL << i)) - val = cpu_if->vgic_lr[i]; + if (!(live_lrs & (1UL << i))) + continue; - writel_relaxed(val, base + GICH_LR0 + (i * 4)); + writel_relaxed(cpu_if->vgic_lr[i], + base + GICH_LR0 + (i * 4)); } } -- cgit v1.1 From 667a87a928c9e1939b1a518be0d62b24378c1fe8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 9 Feb 2016 17:37:39 +0000 Subject: KVM: arm/arm64: vgic-v2: Make GICD_SGIR quicker to hit The GICD_SGIR register lives a long way from the beginning of the handler array, which is searched linearly. As this is hit pretty often, let's move it up. This saves us some precious cycles when the guest is generating IPIs. Acked-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/vgic-v2-emul.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/vgic-v2-emul.c b/virt/kvm/arm/vgic-v2-emul.c index 1390797..1b0bee0 100644 --- a/virt/kvm/arm/vgic-v2-emul.c +++ b/virt/kvm/arm/vgic-v2-emul.c @@ -321,6 +321,11 @@ static bool handle_mmio_sgi_clear(struct kvm_vcpu *vcpu, static const struct vgic_io_range vgic_dist_ranges[] = { { + .base = GIC_DIST_SOFTINT, + .len = 4, + .handle_mmio = handle_mmio_sgi_reg, + }, + { .base = GIC_DIST_CTRL, .len = 12, .bits_per_irq = 0, @@ -387,11 +392,6 @@ static const struct vgic_io_range vgic_dist_ranges[] = { .handle_mmio = handle_mmio_cfg_reg, }, { - .base = GIC_DIST_SOFTINT, - .len = 4, - .handle_mmio = handle_mmio_sgi_reg, - }, - { .base = GIC_DIST_SGI_PENDING_CLEAR, .len = VGIC_NR_SGIS, .handle_mmio = handle_mmio_sgi_clear, -- cgit v1.1 From 1b8e83c04ee2c05c0cd0d304c4b389adf24ebe7f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 17 Feb 2016 10:25:05 +0000 Subject: arm64: KVM: vgic-v3: Avoid accessing ICH registers Just like on GICv2, we're a bit hammer-happy with GICv3, and access them more often than we should. Adopt a policy similar to what we do for GICv2, only save/restoring the minimal set of registers. As we don't access the registers linearly anymore (we may skip some), the convoluted accessors become slightly simpler, and we can drop the ugly indexing macro that tended to confuse the reviewers. Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/vgic-v3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'virt') diff --git a/virt/kvm/arm/vgic-v3.c b/virt/kvm/arm/vgic-v3.c index 453eafd..11b5ff6 100644 --- a/virt/kvm/arm/vgic-v3.c +++ b/virt/kvm/arm/vgic-v3.c @@ -42,7 +42,7 @@ static u32 ich_vtr_el2; static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr) { struct vgic_lr lr_desc; - u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[VGIC_V3_LR_INDEX(lr)]; + u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr]; if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) lr_desc.irq = val & ICH_LR_VIRTUALID_MASK; @@ -106,7 +106,7 @@ static void vgic_v3_set_lr(struct kvm_vcpu *vcpu, int lr, lr_val |= ((u64)lr_desc.hwirq) << ICH_LR_PHYS_ID_SHIFT; } - vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[VGIC_V3_LR_INDEX(lr)] = lr_val; + vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = lr_val; if (!(lr_desc.state & LR_STATE_MASK)) vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr |= (1U << lr); -- cgit v1.1 From 0d98d00b8d80bfdee95cf7e85f20f107377e2662 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 3 Mar 2016 15:43:58 +0000 Subject: arm64: KVM: vgic-v3: Reset LRs at boot time In order to let the GICv3 code be more lazy in the way it accesses the LRs, it is necessary to start with a clean slate. Let's reset the LRs on each CPU when the vgic is probed (which includes a round trip to EL2...). Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier --- virt/kvm/arm/vgic-v3.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'virt') diff --git a/virt/kvm/arm/vgic-v3.c b/virt/kvm/arm/vgic-v3.c index 11b5ff6..999bdc6 100644 --- a/virt/kvm/arm/vgic-v3.c +++ b/virt/kvm/arm/vgic-v3.c @@ -216,6 +216,11 @@ static const struct vgic_ops vgic_v3_ops = { static struct vgic_params vgic_v3_params; +static void vgic_cpu_init_lrs(void *params) +{ + kvm_call_hyp(__vgic_v3_init_lrs); +} + /** * vgic_v3_probe - probe for a GICv3 compatible interrupt controller in DT * @node: pointer to the DT node @@ -284,6 +289,8 @@ int vgic_v3_probe(struct device_node *vgic_node, kvm_info("%s@%llx IRQ%d\n", vgic_node->name, vcpu_res.start, vgic->maint_irq); + on_each_cpu(vgic_cpu_init_lrs, vgic, 1); + *ops = &vgic_v3_ops; *params = vgic; -- cgit v1.1