diff options
author | kib <kib@FreeBSD.org> | 2013-09-04 23:31:29 +0000 |
---|---|---|
committer | kib <kib@FreeBSD.org> | 2013-09-04 23:31:29 +0000 |
commit | 23ba68da73b54f9fe365d3131a80b52961a4c767 (patch) | |
tree | 3f3ca1a974047e79f378a04299da4a474e55670c /sys/amd64/amd64/mp_machdep.c | |
parent | 99fb6f77e5b8c132aeb7ed08996b210789c5e3aa (diff) | |
download | FreeBSD-src-23ba68da73b54f9fe365d3131a80b52961a4c767.zip FreeBSD-src-23ba68da73b54f9fe365d3131a80b52961a4c767.tar.gz |
Tidy up some loose ends in the PCID code:
- Restore the pre-PCID TLB shootdown handlers for whole address space
and single page invalidation asm code, and assign the IPI handler to
them when PCID is not supported or disabled. Old handlers have
linear control flow. But, still use the common return sequence.
- Stop using pcpu for INVPCID descriptors in the invlrg handler. It
is enough to allocate descriptors on the stack. As result, two
SWAPGS instructions are shaved off from the code for Haswell+.
- Fix the reverted condition in invlrng for checking of the PCID
support [1], also in invlrng check that pmap is kernel pmap before
performing other tests. For the kernel pmap, which provides global
mappings, the INVLPG must be used for invalidation always.
- Save the pre-computed pmap' %CR3 register in the struct pmap. This
allows to remove several checks for pm_pcid validity when %CR3 is
reloaded [2].
Noted by: gibbs [1]
Discussed with: alc [2]
Tested by: pho, flo
Sponsored by: The FreeBSD Foundation
Diffstat (limited to 'sys/amd64/amd64/mp_machdep.c')
-rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 21 |
1 files changed, 13 insertions, 8 deletions
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 530aa61..3addd43 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -127,6 +127,8 @@ static u_long *ipi_hardclock_counts[MAXCPU]; extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); +extern int pmap_pcid_enabled; + /* * Local data and functions. */ @@ -524,8 +526,15 @@ cpu_mp_start(void) } /* Install an inter-CPU IPI for TLB invalidation */ - setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0); - setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0); + if (pmap_pcid_enabled) { + setidt(IPI_INVLTLB, IDTVEC(invltlb_pcid), SDT_SYSIGT, + SEL_KPL, 0); + setidt(IPI_INVLPG, IDTVEC(invlpg_pcid), SDT_SYSIGT, + SEL_KPL, 0); + } else { + setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0); + setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0); + } setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0); /* Install an inter-CPU IPI for cache invalidation. */ @@ -605,8 +614,6 @@ cpu_mp_announce(void) } } -extern int pmap_pcid_enabled; - /* * AP CPU's call this to initialize themselves. */ @@ -1141,8 +1148,7 @@ smp_tlb_shootdown(u_int vector, pmap_t pmap, vm_offset_t addr1, smp_tlb_invpcid.pcid = 0; } else { smp_tlb_invpcid.pcid = pmap->pm_pcid; - pcid_cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4) | - (pmap->pm_pcid == -1 ? 0 : pmap->pm_pcid); + pcid_cr3 = pmap->pm_cr3; } smp_tlb_addr2 = addr2; smp_tlb_pmap = pmap; @@ -1176,8 +1182,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap, smp_tlb_invpcid.pcid = 0; } else { smp_tlb_invpcid.pcid = pmap->pm_pcid; - pcid_cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4) | - (pmap->pm_pcid == -1 ? 0 : pmap->pm_pcid); + pcid_cr3 = pmap->pm_cr3; } smp_tlb_addr2 = addr2; smp_tlb_pmap = pmap; |