diff options
author | kib <kib@FreeBSD.org> | 2016-10-03 09:41:33 +0000 |
---|---|---|
committer | kib <kib@FreeBSD.org> | 2016-10-03 09:41:33 +0000 |
commit | 798338843f10f02a3bd77ebb094baec9ffe1a315 (patch) | |
tree | 763eb42636dbca2e03b5282c760ee65948420e32 /sys/amd64/amd64/pmap.c | |
parent | 3c9e39ca85ea89ab37b7638f5bbdc89ab33bc0a9 (diff) | |
download | FreeBSD-src-798338843f10f02a3bd77ebb094baec9ffe1a315.zip FreeBSD-src-798338843f10f02a3bd77ebb094baec9ffe1a315.tar.gz |
MFC r306350:
For machines which support PCID but not have INVPCID instruction,
i.e. SandyBridge and IvyBridge, correct a race between pmap_activate()
and invltlb_pcid_handler().
Diffstat (limited to 'sys/amd64/amd64/pmap.c')
-rw-r--r-- | sys/amd64/amd64/pmap.c | 30 |
1 files changed, 29 insertions, 1 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 7063827..ae525da 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -6868,6 +6868,7 @@ pmap_activate_sw(struct thread *td) { pmap_t oldpmap, pmap; uint64_t cached, cr3; + register_t rflags; u_int cpuid; oldpmap = PCPU_GET(curpmap); @@ -6891,16 +6892,43 @@ pmap_activate_sw(struct thread *td) pmap == kernel_pmap, ("non-kernel pmap thread %p pmap %p cpu %d pcid %#x", td, pmap, cpuid, pmap->pm_pcids[cpuid].pm_pcid)); + + /* + * If the INVPCID instruction is not available, + * invltlb_pcid_handler() is used for handle + * invalidate_all IPI, which checks for curpmap == + * smp_tlb_pmap. Below operations sequence has a + * window where %CR3 is loaded with the new pmap's + * PML4 address, but curpmap value is not yet updated. + * This causes invltlb IPI handler, called between the + * updates, to execute as NOP, which leaves stale TLB + * entries. + * + * Note that the most typical use of + * pmap_activate_sw(), from the context switch, is + * immune to this race, because interrupts are + * disabled (while the thread lock is owned), and IPI + * happends after curpmap is updated. Protect other + * callers in a similar way, by disabling interrupts + * around the %cr3 register reload and curpmap + * assignment. + */ + if (!invpcid_works) + rflags = intr_disable(); + if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | cached); if (cached) PCPU_INC(pm_save_cnt); } + PCPU_SET(curpmap, pmap); + if (!invpcid_works) + intr_restore(rflags); } else if (cr3 != pmap->pm_cr3) { load_cr3(pmap->pm_cr3); + PCPU_SET(curpmap, pmap); } - PCPU_SET(curpmap, pmap); #ifdef SMP CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); #else |