diff options
author | attilio <attilio@FreeBSD.org> | 2011-07-04 12:04:52 +0000 |
---|---|---|
committer | attilio <attilio@FreeBSD.org> | 2011-07-04 12:04:52 +0000 |
commit | 364d0522f778b206262efce0932d6cea821879c6 (patch) | |
tree | f85bc4cd83b575a0c42f08d746e644f6dfc0e2ef /sys/i386 | |
parent | 95ca970257de274a90a1a867048c5ace5acf532d (diff) | |
download | FreeBSD-src-364d0522f778b206262efce0932d6cea821879c6.zip FreeBSD-src-364d0522f778b206262efce0932d6cea821879c6.tar.gz |
With retirement of cpumask_t and usage of cpuset_t for representing a
mask of CPUs, pc_other_cpus and pc_cpumask become highly inefficient.
Remove them and replace their usage with custom pc_cpuid magic (as,
atm, pc_cpumask can be easilly represented by (1 << pc_cpuid) and
pc_other_cpus by (all_cpus & ~(1 << pc_cpuid))).
This change is not targeted for MFC because of struct pcpu members
removal and dependency by cpumask_t retirement.
MD review by: marcel, marius, alc
Tested by: pluknet
MD testing by: marcel, marius, gonzo, andreast
Diffstat (limited to 'sys/i386')
-rw-r--r-- | sys/i386/i386/mp_machdep.c | 60 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 90 | ||||
-rw-r--r-- | sys/i386/i386/vm_machdep.c | 20 | ||||
-rw-r--r-- | sys/i386/xen/mp_machdep.c | 54 | ||||
-rw-r--r-- | sys/i386/xen/pmap.c | 53 |
5 files changed, 125 insertions, 152 deletions
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 78c90c0..0900901 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -658,12 +658,11 @@ cpu_mp_announce(void) void init_secondary(void) { - cpuset_t tcpuset, tallcpus; struct pcpu *pc; vm_offset_t addr; int gsel_tss; int x, myid; - u_int cr0; + u_int cpuid, cr0; /* bootAP is set in start_ap() to our ID. */ myid = bootAP; @@ -758,8 +757,9 @@ init_secondary(void) #endif /* A quick check from sanity claus */ + cpuid = PCPU_GET(cpuid); if (PCPU_GET(apic_id) != lapic_id()) { - printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); + printf("SMP: cpuid = %d\n", cpuid); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); panic("cpuid mismatch! boom!!"); @@ -781,19 +781,13 @@ init_secondary(void) smp_cpus++; - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - tcpuset = PCPU_GET(cpumask); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); + printf("SMP: AP CPU #%d Launched!\n", cpuid); /* Determine if we are a logical CPU. */ /* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */ if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0) - CPU_OR(&logical_cpus_mask, &tcpuset); - - /* Build our map of 'other' CPUs. */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, &tcpuset); - PCPU_SET(other_cpus, tallcpus); + CPU_SET(cpuid, &logical_cpus_mask); if (bootverbose) lapic_dump("AP"); @@ -934,7 +928,6 @@ assign_cpu_ids(void) static int start_all_aps(void) { - cpuset_t tallcpus; #ifndef PC98 u_char mpbiosreason; #endif @@ -997,11 +990,6 @@ start_all_aps(void) CPU_SET(cpu, &all_cpus); /* record AP in CPU map */ } - /* build our map of 'other' CPUs */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, PCPU_PTR(cpumask)); - PCPU_SET(other_cpus, tallcpus); - /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; @@ -1253,9 +1241,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_of if (othercpus < 1) return; } else { - sched_pin(); - CPU_NAND(&mask, PCPU_PTR(cpumask)); - sched_unpin(); + CPU_CLR(PCPU_GET(cpuid), &mask); if (CPU_EMPTY(&mask)) return; } @@ -1452,11 +1438,12 @@ ipi_cpu(int cpu, u_int ipi) void ipi_all_but_self(u_int ipi) { + cpuset_t other_cpus; - sched_pin(); + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); if (IPI_IS_BITMAPED(ipi)) { - ipi_selected(PCPU_GET(other_cpus), ipi); - sched_unpin(); + ipi_selected(other_cpus, ipi); return; } @@ -1466,8 +1453,7 @@ ipi_all_but_self(u_int ipi) * Set the mask of receiving CPUs for this purpose. */ if (ipi == IPI_STOP_HARD) - CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus)); - sched_unpin(); + CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); @@ -1476,7 +1462,7 @@ ipi_all_but_self(u_int ipi) int ipi_nmi_handler() { - cpuset_t cpumask; + u_int cpuid; /* * As long as there is not a simple way to know about a NMI's @@ -1484,13 +1470,11 @@ ipi_nmi_handler() * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ - sched_pin(); - cpumask = PCPU_GET(cpumask); - sched_unpin(); - if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) return (1); - CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); + CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); cpustop_handler(); return (0); } @@ -1502,25 +1486,21 @@ ipi_nmi_handler() void cpustop_handler(void) { - cpuset_t cpumask; u_int cpu; - sched_pin(); cpu = PCPU_GET(cpuid); - cpumask = PCPU_GET(cpumask); - sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index e63cddd..002e529 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -932,16 +932,18 @@ pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde) void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invlpg(va); smp_invlpg(va); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invlpg(va); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -953,8 +955,9 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va) void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; vm_offset_t addr; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { @@ -962,9 +965,10 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) invlpg(addr); smp_invlpg_range(sva, eva); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); CPU_AND(&other_cpus, &pmap->pm_active); @@ -977,16 +981,18 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) void pmap_invalidate_all(pmap_t pmap) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invltlb(); smp_invltlb(); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -1006,11 +1012,11 @@ pmap_invalidate_cache(void) } struct pde_action { - cpuset_t store; /* processor that updates the PDE */ cpuset_t invalidate; /* processors that invalidate their TLB */ vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; + u_int store; /* processor that updates the PDE */ }; static void @@ -1020,9 +1026,7 @@ pmap_update_pde_kernel(void *arg) pd_entry_t *pde; pmap_t pmap; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) { /* * Elsewhere, this operation requires allpmaps_lock for @@ -1033,8 +1037,7 @@ pmap_update_pde_kernel(void *arg) pde = pmap_pde(pmap, act->va); pde_store(pde, act->newpde); } - } else - sched_unpin(); + } } static void @@ -1042,12 +1045,8 @@ pmap_update_pde_user(void *arg) { struct pde_action *act = arg; - sched_pin(); - if (!CPU_CMP(&act->store, PCPU_PTR(cpumask))) { - sched_unpin(); + if (act->store == PCPU_GET(cpuid)) pde_store(act->pde, act->newpde); - } else - sched_unpin(); } static void @@ -1055,12 +1054,8 @@ pmap_update_pde_teardown(void *arg) { struct pde_action *act = arg; - sched_pin(); - if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) { - sched_unpin(); + if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate)) pmap_update_pde_invalidate(act->va, act->newpde); - } else - sched_unpin(); } /* @@ -1075,23 +1070,25 @@ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; - cpuset_t active, cpumask, other_cpus; + cpuset_t active, other_cpus; + u_int cpuid; sched_pin(); - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); if (pmap == kernel_pmap) active = all_cpus; else active = pmap->pm_active; if (CPU_OVERLAP(&active, &other_cpus)) { - act.store = cpumask; + act.store = cpuid; act.invalidate = active; act.va = va; act.pde = pde; act.newpde = newpde; - CPU_OR(&cpumask, &active); - smp_rendezvous_cpus(cpumask, + CPU_SET(cpuid, &active); + smp_rendezvous_cpus(active, smp_no_rendevous_barrier, pmap == kernel_pmap ? pmap_update_pde_kernel : pmap_update_pde_user, pmap_update_pde_teardown, &act); @@ -1100,7 +1097,7 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) pmap_kenter_pde(va, newpde); else pde_store(pde, newpde); - if (CPU_OVERLAP(&active, &cpumask)) + if (CPU_ISSET(cpuid, &active)) pmap_update_pde_invalidate(va, newpde); } sched_unpin(); @@ -1925,12 +1922,12 @@ pmap_lazyfix_action(void) } static void -pmap_lazyfix_self(cpuset_t mymask) +pmap_lazyfix_self(u_int cpuid) { if (rcr3() == lazyptd) load_cr3(PCPU_GET(curpcb)->pcb_cr3); - CPU_NAND_ATOMIC(lazymask, &mymask); + CPU_CLR_ATOMIC(cpuid, lazymask); } @@ -1938,7 +1935,7 @@ static void pmap_lazyfix(pmap_t pmap) { cpuset_t mymask, mask; - u_int spins; + u_int cpuid, spins; int lsb; mask = pmap->pm_active; @@ -1956,10 +1953,13 @@ pmap_lazyfix(pmap_t pmap) #else lazyptd = vtophys(pmap->pm_pdir); #endif - mymask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + + /* Use a cpuset just for having an easy check. */ + CPU_SETOF(cpuid, &mymask); if (!CPU_CMP(&mask, &mymask)) { lazymask = &pmap->pm_active; - pmap_lazyfix_self(mymask); + pmap_lazyfix_self(cpuid); } else { atomic_store_rel_int((u_int *)&lazymask, (u_int)&pmap->pm_active); @@ -5098,17 +5098,19 @@ void pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + u_int cpuid; u_int32_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + cpuid = PCPU_GET(cpuid); #if defined(SMP) - CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); + CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else - CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR(cpuid, &oldpmap->pm_active); + CPU_SET(cpuid, &pmap->pm_active); #endif #ifdef PAE cr3 = vtophys(pmap->pm_pdpt); diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index a084e09..0a7bc21 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -602,8 +602,8 @@ cpu_reset() u_int cnt; if (smp_active) { - sched_pin(); - map = PCPU_GET(other_cpus); + map = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &map); CPU_NAND(&map, &stopped_cpus); if (!CPU_EMPTY(&map)) { printf("cpu_reset: Stopping other CPUs\n"); @@ -612,7 +612,6 @@ cpu_reset() if (PCPU_GET(cpuid) != 0) { cpu_reset_proxyid = PCPU_GET(cpuid); - sched_unpin(); cpustop_restartfunc = cpu_reset_proxy; cpu_reset_proxy_active = 0; printf("cpu_reset: Restarting BSP\n"); @@ -632,8 +631,7 @@ cpu_reset() while (1); /* NOTREACHED */ - } else - sched_unpin(); + } DELAY(1000000); } @@ -802,7 +800,8 @@ sf_buf_alloc(struct vm_page *m, int flags) struct sf_head *hash_list; struct sf_buf *sf; #ifdef SMP - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; #endif int error; @@ -877,13 +876,14 @@ sf_buf_alloc(struct vm_page *m, int flags) CPU_ZERO(&sf->cpumask); shootdown: sched_pin(); - cpumask = PCPU_GET(cpumask); - if (!CPU_OVERLAP(&cpumask, &sf->cpumask)) { - CPU_OR(&sf->cpumask, &cpumask); + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &sf->cpumask)) { + CPU_SET(cpuid, &sf->cpumask); invlpg(sf->kva); } if ((flags & SFB_CPUPRIVATE) == 0) { - other_cpus = PCPU_GET(other_cpus); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); CPU_NAND(&other_cpus, &sf->cpumask); if (!CPU_EMPTY(&other_cpus)) { CPU_OR(&sf->cpumask, &other_cpus); diff --git a/sys/i386/xen/mp_machdep.c b/sys/i386/xen/mp_machdep.c index 2d05596..a0ef1e8 100644 --- a/sys/i386/xen/mp_machdep.c +++ b/sys/i386/xen/mp_machdep.c @@ -523,8 +523,8 @@ xen_smp_intr_init_cpus(void *unused) void init_secondary(void) { - cpuset_t tcpuset, tallcpus; vm_offset_t addr; + u_int cpuid; int gsel_tss; @@ -601,23 +601,18 @@ init_secondary(void) #endif smp_cpus++; - CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); - tcpuset = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid); + printf("SMP: AP CPU #%d Launched!\n", cpuid); /* Determine if we are a logical CPU. */ if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) - CPU_OR(&logical_cpus_mask, &tcpuset); + CPU_SET(cpuid, &logical_cpus_mask); /* Determine if we are a hyperthread. */ if (hyperthreading_cpus > 1 && PCPU_GET(apic_id) % hyperthreading_cpus != 0) - CPU_OR(&hyperthreading_cpus_mask, &tcpuset); - - /* Build our map of 'other' CPUs. */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, &tcpuset); - PCPU_SET(other_cpus, tallcpus); + CPU_SET(cpuid, &hyperthreading_cpus_mask); #if 0 if (bootverbose) lapic_dump("AP"); @@ -731,7 +726,6 @@ assign_cpu_ids(void) int start_all_aps(void) { - cpuset_t tallcpus; int x,apic_id, cpu; struct pcpu *pc; @@ -789,11 +783,6 @@ start_all_aps(void) } - /* build our map of 'other' CPUs */ - tallcpus = all_cpus; - CPU_NAND(&tallcpus, PCPU_PTR(cpumask)); - PCPU_SET(other_cpus, tallcpus); - pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); /* number of APs actually started */ @@ -1031,9 +1020,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_of if (othercpus < 1) return; } else { - critical_enter(); - CPU_NAND(&mask, PCPU_PTR(cpumask)); - critical_exit(); + CPU_CLR(PCPU_GET(cpuid), &mask); if (CPU_EMPTY(&mask)) return; } @@ -1184,9 +1171,8 @@ ipi_all_but_self(u_int ipi) * of help in order to understand what is the source. * Set the mask of receiving CPUs for this purpose. */ - sched_pin(); - other_cpus = PCPU_GET(other_cpus); - sched_unpin(); + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); if (ipi == IPI_STOP_HARD) CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus); @@ -1197,7 +1183,7 @@ ipi_all_but_self(u_int ipi) int ipi_nmi_handler() { - cpuset_t cpumask; + u_int cpuid; /* * As long as there is not a simple way to know about a NMI's @@ -1205,13 +1191,11 @@ ipi_nmi_handler() * the global pending bitword an IPI_STOP_HARD has been issued * and should be handled. */ - sched_pin(); - cpumask = PCPU_GET(cpumask); - sched_unpin(); - if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask)) + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &ipi_nmi_pending)) return (1); - CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask); + CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending); cpustop_handler(); return (0); } @@ -1223,25 +1207,21 @@ ipi_nmi_handler() void cpustop_handler(void) { - cpuset_t cpumask; int cpu; - sched_pin(); - cpumask = PCPU_GET(cpumask); cpu = PCPU_GET(cpuid); - sched_unpin(); savectx(&stoppcbs[cpu]); /* Indicate that we are stopped */ - CPU_OR_ATOMIC(&stopped_cpus, &cpumask); + CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ - while (!CPU_OVERLAP(&started_cpus, &cpumask)) + while (!CPU_ISSET(cpu, &started_cpus)) ia32_pause(); - CPU_NAND_ATOMIC(&started_cpus, &cpumask); - CPU_NAND_ATOMIC(&stopped_cpus, &cpumask); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); if (cpu == 0 && cpustop_restartfunc != NULL) { cpustop_restartfunc(); diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c index 87bc0a0..b5e71da 100644 --- a/sys/i386/xen/pmap.c +++ b/sys/i386/xen/pmap.c @@ -802,7 +802,8 @@ pmap_cache_bits(int mode, boolean_t is_pde) void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x", pmap, va); @@ -812,9 +813,10 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va) invlpg(va); smp_invlpg(va); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invlpg(va); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -827,8 +829,9 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va) void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; vm_offset_t addr; + u_int cpuid; CTR3(KTR_PMAP, "pmap_invalidate_page: pmap=%p eva=0x%x sva=0x%x", pmap, sva, eva); @@ -839,9 +842,10 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) invlpg(addr); smp_invlpg_range(sva, eva); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); CPU_AND(&other_cpus, &pmap->pm_active); @@ -855,7 +859,8 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) void pmap_invalidate_all(pmap_t pmap) { - cpuset_t cpumask, other_cpus; + cpuset_t other_cpus; + u_int cpuid; CTR1(KTR_PMAP, "pmap_invalidate_page: pmap=%p", pmap); @@ -864,9 +869,10 @@ pmap_invalidate_all(pmap_t pmap) invltlb(); smp_invltlb(); } else { - cpumask = PCPU_GET(cpumask); - other_cpus = PCPU_GET(other_cpus); - if (CPU_OVERLAP(&pmap->pm_active, &cpumask)) + cpuid = PCPU_GET(cpuid); + other_cpus = all_cpus; + CPU_CLR(cpuid, &other_cpus); + if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); CPU_AND(&other_cpus, &pmap->pm_active); if (!CPU_EMPTY(&other_cpus)) @@ -1708,12 +1714,12 @@ pmap_lazyfix_action(void) } static void -pmap_lazyfix_self(cpuset_t mymask) +pmap_lazyfix_self(u_int cpuid) { if (rcr3() == lazyptd) load_cr3(PCPU_GET(curpcb)->pcb_cr3); - CPU_NAND_ATOMIC(lazymask, &mymask); + CPU_CLR_ATOMIC(cpuid, lazymask); } @@ -1721,7 +1727,7 @@ static void pmap_lazyfix(pmap_t pmap) { cpuset_t mymask, mask; - u_int spins; + u_int cpuid, spins; int lsb; mask = pmap->pm_active; @@ -1739,10 +1745,13 @@ pmap_lazyfix(pmap_t pmap) #else lazyptd = vtophys(pmap->pm_pdir); #endif - mymask = PCPU_GET(cpumask); + cpuid = PCPU_GET(cpuid); + + /* Use a cpuset just for having an easy check. */ + CPU_SETOF(cpuid, &mymask); if (!CPU_CMP(&mask, &mymask)) { lazymask = &pmap->pm_active; - pmap_lazyfix_self(mymask); + pmap_lazyfix_self(cpuid); } else { atomic_store_rel_int((u_int *)&lazymask, (u_int)&pmap->pm_active); @@ -4126,17 +4135,19 @@ void pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; + u_int cpuid; u_int32_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); + cpuid = PCPU_GET(cpuid); #if defined(SMP) - CPU_NAND_ATOMIC(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR_ATOMIC(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); + CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else - CPU_NAND(&oldpmap->pm_active, PCPU_PTR(cpumask)); - CPU_OR(&pmap->pm_active, PCPU_PTR(cpumask)); + CPU_CLR(cpuid, &oldpmap->pm_active); + CPU_SET(cpuid, &pmap->pm_active); #endif #ifdef PAE cr3 = vtophys(pmap->pm_pdpt); |