diff options
| -rw-r--r-- | sys/amd64/amd64/apic_vector.S | 252 | ||||
| -rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 170 | ||||
| -rw-r--r-- | sys/amd64/include/smp.h | 6 | ||||
| -rw-r--r-- | sys/i386/i386/apic_vector.s | 145 | ||||
| -rw-r--r-- | sys/i386/i386/mp_machdep.c | 66 | ||||
| -rw-r--r-- | sys/i386/include/smp.h | 4 | ||||
| -rw-r--r-- | sys/i386/xen/mp_machdep.c | 25 | ||||
| -rw-r--r-- | sys/x86/xen/hvm.c | 193 |
8 files changed, 322 insertions, 539 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index d002b4d..4c87b4e 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -159,101 +159,26 @@ IDTVEC(xen_intr_upcall) #define NAKE_INTR_CS 24 SUPERALIGN_TEXT -global_invltlb: - movq %cr4,%rax - andq $~0x80,%rax /* PGE */ - movq %rax,%cr4 - orq $0x80,%rax - movq %rax,%cr4 -invltlb_ret_clear_pm_save: - movq smp_tlb_pmap,%rdx - testq %rdx,%rdx - jz invltlb_ret_rdx - testb $SEL_RPL_MASK,NAKE_INTR_CS(%rsp) - jz 1f - swapgs -1: - movl PCPU(CPUID),%eax - jz 2f - swapgs -2: - LK btcl %eax,PM_SAVE(%rdx) - SUPERALIGN_TEXT -invltlb_ret_rdx: - popq %rdx -invltlb_ret_rax: +invltlb_ret: movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - LK incl smp_tlb_wait - popq %rax + POP_FRAME jmp doreti_iret SUPERALIGN_TEXT IDTVEC(invltlb_pcid) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invltlb_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - pushq %rdx - - movq %cr3,%rax - - movq $smp_tlb_invpcid,%rdx - cmpl $0,(%rdx) - je global_invltlb - cmpl $-1,(%rdx) - je global_invltlb - - /* - * Only invalidate TLB for entries with current PCID. - */ - cmpl $0,invpcid_works - je 1f - /* Use invpcid if available. */ - movl $1,%eax /* INVPCID_CTX */ - /* invpcid (%rdx),%rax */ - .byte 0x66,0x0f,0x38,0x82,0x02 - jmp invltlb_ret_clear_pm_save -1: - /* Otherwise reload %cr3 twice. */ - movq pcid_cr3,%rdx - cmpq %rax,%rdx - je 2f - movq %rdx,%cr3 /* Invalidate, bit 63 is zero. */ - btsq $63,%rax -2: - movq %rax,%cr3 - jmp invltlb_ret_clear_pm_save + call invltlb_pcid_handler + jmp invltlb_ret + SUPERALIGN_TEXT IDTVEC(invltlb) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invltlb_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - movq %cr3, %rax /* invalidate the TLB */ - movq %rax, %cr3 - jmp invltlb_ret_rax + call invltlb_handler + jmp invltlb_ret /* * Single page TLB shootdown @@ -261,86 +186,17 @@ IDTVEC(invltlb) .text SUPERALIGN_TEXT IDTVEC(invlpg_pcid) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlpg_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - pushq %rdx - movq $smp_tlb_invpcid,%rdx - cmpl $0,invpcid_works - jne 2f - - /* kernel pmap - use invlpg to invalidate global mapping */ - cmpl $0,(%rdx) - je 3f - cmpl $-1,(%rdx) - je global_invltlb - - /* - * PCID supported, but INVPCID is not. - * Temporarily switch to the target address space and do INVLPG. - */ - pushq %rcx - movq %cr3,%rcx - movq pcid_cr3,%rax - cmp %rcx,%rax - je 1f - btsq $63,%rax - movq %rax,%cr3 -1: movq 8(%rdx),%rax - invlpg (%rax) - btsq $63,%rcx - movq %rcx,%cr3 - popq %rcx - jmp invltlb_ret_rdx - - /* - * Invalidate the TLB entry using INVPCID_ADDR. - */ -2: - xorl %eax,%eax -/* invpcid (%rdx),%rax */ - .byte 0x66,0x0f,0x38,0x82,0x02 - jmp invltlb_ret_rdx - - /* - * PCID is not supported or kernel pmap. - * Invalidate single page using INVLPG. - */ -3: - movq 8(%rdx),%rax - invlpg (%rax) - jmp invltlb_ret_rdx + call invlpg_pcid_handler + jmp invltlb_ret SUPERALIGN_TEXT IDTVEC(invlpg) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlpg_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - movq smp_tlb_invpcid+8,%rax - invlpg (%rax) /* invalidate single page */ - jmp invltlb_ret_rax + call invlpg_handler + jmp invltlb_ret /* * Page range TLB shootdown. @@ -348,81 +204,10 @@ IDTVEC(invlpg) .text SUPERALIGN_TEXT IDTVEC(invlrng) -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) PUSH_FRAME - movl PCPU(CPUID), %eax -#ifdef COUNT_XINVLTLB_HITS - incl xhits_rng(,%rax,4) -#endif -#ifdef COUNT_IPIS - movq ipi_invlrng_counts(,%rax,8),%rax - incq (%rax) -#endif - POP_FRAME -#endif - pushq %rax - pushq %rdx - movq $smp_tlb_invpcid,%rdx - cmpl $0,pmap_pcid_enabled - je invlrng_single_page - - /* kernel pmap - use invlpg to invalidate global mapping */ - cmpl $0,(%rdx) - je invlrng_single_page - cmpl $-1,(%rdx) - je global_invltlb - cmpl $0,invpcid_works - jne invlrng_invpcid - - pushq %rcx - movq %cr3,%rcx - movq pcid_cr3,%rax - cmpq %rcx,%rax - je 1f - btsq $63,%rax - movq %rax,%cr3 -1: - movq 8(%rdx),%rdx - movq smp_tlb_addr2,%rax -2: - invlpg (%rdx) - addq $PAGE_SIZE,%rdx - cmpq %rax,%rdx - jb 2b - btsq $63,%rcx - movq %rcx,%cr3 - popq %rcx - jmp invltlb_ret_rdx - -invlrng_invpcid: - pushq %rcx - subq $16,%rsp - movq (%rdx),%rcx - movq %rcx,(%rsp) - movq 8(%rdx),%rax - movq %rax,8(%rsp) - movq smp_tlb_addr2,%rcx - subq %rax,%rcx - shrq $PAGE_SHIFT,%rcx -1: -// invpcid (%rdx),%rax - .byte 0x66,0x0f,0x38,0x82,0x02 - addq $PAGE_SIZE,8(%rsp) - dec %rcx - jne 1b - addq $16,%rsp - popq %rcx - jmp invltlb_ret_rdx - -invlrng_single_page: - movq 8(%rdx),%rdx - movq smp_tlb_addr2,%rax -1: invlpg (%rdx) /* invalidate single page */ - addq $PAGE_SIZE,%rdx - cmpq %rax,%rdx - jb 1b - jmp invltlb_ret_rdx + call invlrng_handler + jmp invltlb_ret /* * Invalidate cache. @@ -430,17 +215,10 @@ invlrng_single_page: .text SUPERALIGN_TEXT IDTVEC(invlcache) -#ifdef COUNT_IPIS PUSH_FRAME - movl PCPU(CPUID), %eax - movq ipi_invlcache_counts(,%rax,8),%rax - incq (%rax) - POP_FRAME -#endif - pushq %rax - wbinvd - jmp invltlb_ret_rax + call invlcache_handler + jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 1b38f60..9f03e4c 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -109,6 +109,7 @@ struct invpcid_descr smp_tlb_invpcid; volatile int smp_tlb_wait; uint64_t pcid_cr3; pmap_t smp_tlb_pmap; +extern int invpcid_works; #ifdef COUNT_IPIS /* Interrupt counts. */ @@ -1496,6 +1497,175 @@ cpususpend_handler(void) } /* + * Handlers for TLB related IPIs + */ +void +invltlb_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invltlb(); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invltlb_pcid_handler(void) +{ + uint64_t cr3; +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + cr3 = rcr3(); + if (smp_tlb_invpcid.pcid != (uint64_t)-1 && + smp_tlb_invpcid.pcid != 0) { + + if (invpcid_works) { + invpcid(&smp_tlb_invpcid, INVPCID_CTX); + } else { + /* Otherwise reload %cr3 twice. */ + if (cr3 != pcid_cr3) { + load_cr3(pcid_cr3); + cr3 |= CR3_PCID_SAVE; + } + load_cr3(cr3); + } + } else { + invltlb_globpcid(); + } + if (smp_tlb_pmap != NULL) + CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invlpg(smp_tlb_invpcid.addr); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_pcid_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + if (invpcid_works) { + invpcid(&smp_tlb_invpcid, INVPCID_ADDR); + } else if (smp_tlb_invpcid.pcid == 0) { + invlpg(smp_tlb_invpcid.addr); + } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { + invltlb_globpcid(); + } else { + uint64_t cr3; + + /* + * PCID supported, but INVPCID is not. + * Temporarily switch to the target address + * space and do INVLPG. + */ + cr3 = rcr3(); + if (cr3 != pcid_cr3) + load_cr3(pcid_cr3 | CR3_PCID_SAVE); + invlpg(smp_tlb_invpcid.addr); + load_cr3(cr3 | CR3_PCID_SAVE); + } + + atomic_add_int(&smp_tlb_wait, 1); +} + +static inline void +invlpg_range(vm_offset_t start, vm_offset_t end) +{ + + do { + invlpg(start); + start += PAGE_SIZE; + } while (start < end); +} + +void +invlrng_handler(void) +{ + vm_offset_t addr; +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + addr = smp_tlb_invpcid.addr; + if (pmap_pcid_enabled) { + if (invpcid_works) { + struct invpcid_descr d; + + d = smp_tlb_invpcid; + do { + invpcid(&d, INVPCID_ADDR); + d.addr += PAGE_SIZE; + } while (d.addr < smp_tlb_addr2); + } else if (smp_tlb_invpcid.pcid == 0) { + /* + * kernel pmap - use invlpg to invalidate + * global mapping. + */ + invlpg_range(addr, smp_tlb_addr2); + } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { + invltlb_globpcid(); + if (smp_tlb_pmap != NULL) { + CPU_CLR_ATOMIC(PCPU_GET(cpuid), + &smp_tlb_pmap->pm_save); + } + } else { + uint64_t cr3; + + cr3 = rcr3(); + if (cr3 != pcid_cr3) + load_cr3(pcid_cr3 | CR3_PCID_SAVE); + invlpg_range(addr, smp_tlb_addr2); + load_cr3(cr3 | CR3_PCID_SAVE); + } + } else { + invlpg_range(addr, smp_tlb_addr2); + } + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlcache_handler(void) +{ +#ifdef COUNT_IPIS + (*ipi_invlcache_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + wbinvd(); + atomic_add_int(&smp_tlb_wait, 1); +} + +/* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index d1b366b..102984c 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -62,6 +62,12 @@ struct pmap; void cpu_add(u_int apic_id, char boot_cpu); void cpustop_handler(void); void cpususpend_handler(void); +void invltlb_handler(void); +void invltlb_pcid_handler(void); +void invlpg_handler(void); +void invlpg_pcid_handler(void); +void invlrng_handler(void); +void invlcache_handler(void); void init_secondary(void); void ipi_startup(int apic_id, int vector); void ipi_all_but_self(u_int ipi); diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index adedbc4..3d1999e 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -163,39 +163,21 @@ IDTVEC(xen_intr_upcall) */ .text SUPERALIGN_TEXT -IDTVEC(invltlb) - pushl %eax - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs -#ifdef COUNT_XINVLTLB_HITS - incl xhits_gbl(,%eax,4) -#endif -#ifdef COUNT_IPIS - movl ipi_invltlb_counts(,%eax,4),%eax - incl (%eax) -#endif -#endif - - movl %cr3, %eax /* invalidate the TLB */ - movl %eax, %cr3 - +invltlb_ret: movl lapic, %eax movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + POP_FRAME + iret + + SUPERALIGN_TEXT +IDTVEC(invltlb) + PUSH_FRAME + SET_KERNEL_SREGS + cld - lock - incl smp_tlb_wait + call invltlb_handler - popl %ds - popl %eax - iret + jmp invltlb_ret /* * Single page TLB shootdown @@ -203,38 +185,13 @@ IDTVEC(invltlb) .text SUPERALIGN_TEXT IDTVEC(invlpg) - pushl %eax - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs -#ifdef COUNT_XINVLTLB_HITS - incl xhits_pg(,%eax,4) -#endif -#ifdef COUNT_IPIS - movl ipi_invlpg_counts(,%eax,4),%eax - incl (%eax) -#endif -#endif - - movl smp_tlb_addr1, %eax - invlpg (%eax) /* invalidate single page */ - - movl lapic, %eax - movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + PUSH_FRAME + SET_KERNEL_SREGS + cld - lock - incl smp_tlb_wait + call invlpg_handler - popl %ds - popl %eax - iret + jmp invltlb_ret /* * Page range TLB shootdown. @@ -242,44 +199,13 @@ IDTVEC(invlpg) .text SUPERALIGN_TEXT IDTVEC(invlrng) - pushl %eax - pushl %edx - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs -#ifdef COUNT_XINVLTLB_HITS - incl xhits_rng(,%eax,4) -#endif -#ifdef COUNT_IPIS - movl ipi_invlrng_counts(,%eax,4),%eax - incl (%eax) -#endif -#endif + PUSH_FRAME + SET_KERNEL_SREGS + cld - movl smp_tlb_addr1, %edx - movl smp_tlb_addr2, %eax -1: invlpg (%edx) /* invalidate single page */ - addl $PAGE_SIZE, %edx - cmpl %eax, %edx - jb 1b + call invlrng_handler - movl lapic, %eax - movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ - - lock - incl smp_tlb_wait - - popl %ds - popl %edx - popl %eax - iret + jmp invltlb_ret /* * Invalidate cache. @@ -287,32 +213,13 @@ IDTVEC(invlrng) .text SUPERALIGN_TEXT IDTVEC(invlcache) - pushl %eax - pushl %ds - movl $KDSEL, %eax /* Kernel data selector */ - movl %eax, %ds - -#ifdef COUNT_IPIS - pushl %fs - movl $KPSEL, %eax /* Private space selector */ - movl %eax, %fs - movl PCPU(CPUID), %eax - popl %fs - movl ipi_invlcache_counts(,%eax,4),%eax - incl (%eax) -#endif - - wbinvd - - movl lapic, %eax - movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + PUSH_FRAME + SET_KERNEL_SREGS + cld - lock - incl smp_tlb_wait + call invlcache_handler - popl %ds - popl %eax - iret + jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 8ee8653..53ac6df 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -1551,6 +1551,72 @@ cpususpend_handler(void) CPU_CLR_ATOMIC(cpu, &suspended_cpus); CPU_CLR_ATOMIC(cpu, &started_cpus); } + +/* + * Handlers for TLB related IPIs + */ +void +invltlb_handler(void) +{ + uint64_t cr3; +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + cr3 = rcr3(); + load_cr3(cr3); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invlpg(smp_tlb_addr1); + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlrng_handler(void) +{ + vm_offset_t addr; +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + addr = smp_tlb_addr1; + do { + invlpg(addr); + addr += PAGE_SIZE; + } while (addr < smp_tlb_addr2); + + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlcache_handler(void) +{ +#ifdef COUNT_IPIS + (*ipi_invlcache_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + wbinvd(); + atomic_add_int(&smp_tlb_wait, 1); +} + /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h index 43dad10..a0936e9 100644 --- a/sys/i386/include/smp.h +++ b/sys/i386/include/smp.h @@ -63,6 +63,10 @@ void cpustop_handler(void); #ifndef XEN void cpususpend_handler(void); #endif +void invltlb_handler(void); +void invlpg_handler(void); +void invlrng_handler(void); +void invlcache_handler(void); void init_secondary(void); void ipi_startup(int apic_id, int vector); void ipi_all_but_self(u_int ipi); diff --git a/sys/i386/xen/mp_machdep.c b/sys/i386/xen/mp_machdep.c index c48fcb2..1bc6572 100644 --- a/sys/i386/xen/mp_machdep.c +++ b/sys/i386/xen/mp_machdep.c @@ -1276,6 +1276,31 @@ cpustop_handler(void) } /* + * Handlers for TLB related IPIs + * + * On i386 Xen PV this are no-ops since this port doesn't support SMP. + */ +void +invltlb_handler(void) +{ +} + +void +invlpg_handler(void) +{ +} + +void +invlrng_handler(void) +{ +} + +void +invlcache_handler(void) +{ +} + +/* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index 72811dc..9f4c6be 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -77,22 +77,12 @@ static void xen_hvm_cpu_resume(void); static void xen_hvm_cpu_init(void); /*---------------------------- Extern Declarations ---------------------------*/ -/* Variables used by mp_machdep to perform the MMU related IPIs */ -extern volatile int smp_tlb_wait; -extern vm_offset_t smp_tlb_addr2; -#ifdef __i386__ -extern vm_offset_t smp_tlb_addr1; -#else -extern struct invpcid_descr smp_tlb_invpcid; -extern uint64_t pcid_cr3; -extern int invpcid_works; -extern int pmap_pcid_enabled; -extern pmap_t smp_tlb_pmap; -#endif - #ifdef __i386__ extern void pmap_lazyfix_action(void); #endif +#ifdef __amd64__ +extern int pmap_pcid_enabled; +#endif /* Variables used by mp_machdep to perform the bitmap IPI */ extern volatile u_int cpu_ipi_pending[MAXCPU]; @@ -179,10 +169,7 @@ static int xen_smp_rendezvous_action(void *arg) { #ifdef COUNT_IPIS - int cpu; - - cpu = PCPU_GET(cpuid); - (*ipi_rendezvous_counts[cpu])++; + (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ smp_rendezvous_action(); @@ -192,20 +179,8 @@ xen_smp_rendezvous_action(void *arg) static int xen_invltlb(void *arg) { -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_gbl[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invltlb_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - invltlb(); - atomic_add_int(&smp_tlb_wait, 1); + invltlb_handler(); return (FILTER_HANDLED); } @@ -213,40 +188,8 @@ xen_invltlb(void *arg) static int xen_invltlb_pcid(void *arg) { - uint64_t cr3; -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_gbl[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invltlb_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - - cr3 = rcr3(); - if (smp_tlb_invpcid.pcid != (uint64_t)-1 && - smp_tlb_invpcid.pcid != 0) { - - if (invpcid_works) { - invpcid(&smp_tlb_invpcid, INVPCID_CTX); - } else { - /* Otherwise reload %cr3 twice. */ - if (cr3 != pcid_cr3) { - load_cr3(pcid_cr3); - cr3 |= CR3_PCID_SAVE; - } - load_cr3(cr3); - } - } else { - invltlb_globpcid(); - } - if (smp_tlb_pmap != NULL) - CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); - atomic_add_int(&smp_tlb_wait, 1); + invltlb_pcid_handler(); return (FILTER_HANDLED); } #endif @@ -254,24 +197,8 @@ xen_invltlb_pcid(void *arg) static int xen_invlpg(void *arg) { -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_pg[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlpg_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - -#ifdef __i386__ - invlpg(smp_tlb_addr1); -#else - invlpg(smp_tlb_invpcid.addr); -#endif - atomic_add_int(&smp_tlb_wait, 1); + invlpg_handler(); return (FILTER_HANDLED); } @@ -279,125 +206,25 @@ xen_invlpg(void *arg) static int xen_invlpg_pcid(void *arg) { -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_pg[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlpg_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ - - if (invpcid_works) { - invpcid(&smp_tlb_invpcid, INVPCID_ADDR); - } else if (smp_tlb_invpcid.pcid == 0) { - invlpg(smp_tlb_invpcid.addr); - } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { - invltlb_globpcid(); - } else { - uint64_t cr3; - - /* - * PCID supported, but INVPCID is not. - * Temporarily switch to the target address - * space and do INVLPG. - */ - cr3 = rcr3(); - if (cr3 != pcid_cr3) - load_cr3(pcid_cr3 | CR3_PCID_SAVE); - invlpg(smp_tlb_invpcid.addr); - load_cr3(cr3 | CR3_PCID_SAVE); - } - atomic_add_int(&smp_tlb_wait, 1); + invlpg_pcid_handler(); return (FILTER_HANDLED); } #endif -static inline void -invlpg_range(vm_offset_t start, vm_offset_t end) -{ - do { - invlpg(start); - start += PAGE_SIZE; - } while (start < end); -} - static int xen_invlrng(void *arg) { - vm_offset_t addr; -#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) - int cpu; - - cpu = PCPU_GET(cpuid); -#ifdef COUNT_XINVLTLB_HITS - xhits_rng[cpu]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlrng_counts[cpu])++; -#endif /* COUNT_IPIS */ -#endif /* COUNT_XINVLTLB_HITS || COUNT_IPIS */ -#ifdef __i386__ - addr = smp_tlb_addr1; - invlpg_range(addr, smp_tlb_addr2); -#else - addr = smp_tlb_invpcid.addr; - if (pmap_pcid_enabled) { - if (invpcid_works) { - struct invpcid_descr d; - - d = smp_tlb_invpcid; - do { - invpcid(&d, INVPCID_ADDR); - d.addr += PAGE_SIZE; - } while (d.addr < smp_tlb_addr2); - } else if (smp_tlb_invpcid.pcid == 0) { - /* - * kernel pmap - use invlpg to invalidate - * global mapping. - */ - invlpg_range(addr, smp_tlb_addr2); - } else if (smp_tlb_invpcid.pcid != (uint64_t)-1) { - invltlb_globpcid(); - if (smp_tlb_pmap != NULL) { - CPU_CLR_ATOMIC(PCPU_GET(cpuid), - &smp_tlb_pmap->pm_save); - } - } else { - uint64_t cr3; - - cr3 = rcr3(); - if (cr3 != pcid_cr3) - load_cr3(pcid_cr3 | CR3_PCID_SAVE); - invlpg_range(addr, smp_tlb_addr2); - load_cr3(cr3 | CR3_PCID_SAVE); - } - } else { - invlpg_range(addr, smp_tlb_addr2); - } -#endif - - atomic_add_int(&smp_tlb_wait, 1); + invlrng_handler(); return (FILTER_HANDLED); } static int xen_invlcache(void *arg) { -#ifdef COUNT_IPIS - int cpu = PCPU_GET(cpuid); - - cpu = PCPU_GET(cpuid); - (*ipi_invlcache_counts[cpu])++; -#endif /* COUNT_IPIS */ - wbinvd(); - atomic_add_int(&smp_tlb_wait, 1); + invlcache_handler(); return (FILTER_HANDLED); } |
