With retirement of cpumask_t and usage of cpuset_t for representing a

mask of CPUs, pc_other_cpus and pc_cpumask become highly inefficient.

Remove them and replace their usage with custom pc_cpuid magic (as,
atm, pc_cpumask can be easilly represented by (1 << pc_cpuid) and
pc_other_cpus by (all_cpus & ~(1 << pc_cpuid))).

This change is not targeted for MFC because of struct pcpu members
removal and dependency by cpumask_t retirement.

MD review by:	marcel, marius, alc
Tested by:	pluknet
MD testing by:	marcel, marius, gonzo, andreast

1 files changed, 25 insertions, 46 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 53988e9..668e79c 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -604,10 +604,10 @@ cpu_mp_announce(void)
 void
 init_secondary(void)
 {
-	cpuset_t tcpuset, tallcpus;
 	struct pcpu *pc;
 	struct nmi_pcpu *np;
 	u_int64_t msr, cr0;
+	u_int cpuid;
 	int cpu, gsel_tss, x;
 	struct region_descriptor ap_gdt;
 
@@ -711,8 +711,9 @@ init_secondary(void)
 	fpuinit();
 
 	/* A quick check from sanity claus */
+	cpuid = PCPU_GET(cpuid);
 	if (PCPU_GET(apic_id) != lapic_id()) {
-		printf("SMP: cpuid = %d\n", PCPU_GET(cpuid));
+		printf("SMP: cpuid = %d\n", cpuid);
 		printf("SMP: actual apic_id = %d\n", lapic_id());
 		printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id));
 		panic("cpuid mismatch! boom!!");
@@ -734,19 +735,13 @@ init_secondary(void)
 
 	smp_cpus++;
 
-	CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
-	printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
-	tcpuset = PCPU_GET(cpumask);
+	CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", cpuid);
+	printf("SMP: AP CPU #%d Launched!\n", cpuid);
 
 	/* Determine if we are a logical CPU. */
 	/* XXX Calculation depends on cpu_logical being a power of 2, e.g. 2 */
 	if (cpu_logical > 1 && PCPU_GET(apic_id) % cpu_logical != 0)
-		CPU_OR(&logical_cpus_mask, &tcpuset);
-
-	/* Build our map of 'other' CPUs. */
-	tallcpus = all_cpus;
-	CPU_NAND(&tallcpus, &tcpuset);
-	PCPU_SET(other_cpus, tallcpus);
+		CPU_SET(cpuid, &logical_cpus_mask);
 
 	if (bootverbose)
 		lapic_dump("AP");
@@ -893,7 +888,6 @@ assign_cpu_ids(void)
 static int
 start_all_aps(void)
 {
-	cpuset_t tallcpus, tcpuset;
 	vm_offset_t va = boot_address + KERNBASE;
 	u_int64_t *pt4, *pt3, *pt2;
 	u_int32_t mpbioswarmvec;
@@ -961,12 +955,6 @@ start_all_aps(void)
 		CPU_SET(cpu, &all_cpus);	/* record AP in CPU map */
 	}
 
-	/* build our map of 'other' CPUs */
-	tallcpus = all_cpus;
-	tcpuset = PCPU_GET(cpumask);
-	CPU_NAND(&tallcpus, &tcpuset);
-	PCPU_SET(other_cpus, tallcpus);
-
 	/* restore the warmstart vector */
 	*(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec;
 
@@ -1150,9 +1138,7 @@ smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_of
 		if (othercpus < 1)
 			return;
 	} else {
-		sched_pin();
-		CPU_NAND(&mask, PCPU_PTR(cpumask));
-		sched_unpin();
+		CPU_CLR(PCPU_GET(cpuid), &mask);
 		if (CPU_EMPTY(&mask))
 			return;
 	}
@@ -1349,11 +1335,13 @@ ipi_cpu(int cpu, u_int ipi)
 void
 ipi_all_but_self(u_int ipi)
 {
+	cpuset_t other_cpus;
+
+	other_cpus = all_cpus;
+	CPU_CLR(PCPU_GET(cpuid), &other_cpus);
 
-	sched_pin();
 	if (IPI_IS_BITMAPED(ipi)) {
-		ipi_selected(PCPU_GET(other_cpus), ipi);
-		sched_unpin();
+		ipi_selected(other_cpus, ipi);
 		return;
 	}
 
@@ -1363,8 +1351,7 @@ ipi_all_but_self(u_int ipi)
 	 * Set the mask of receiving CPUs for this purpose.
 	 */
 	if (ipi == IPI_STOP_HARD)
-		CPU_OR_ATOMIC(&ipi_nmi_pending, PCPU_PTR(other_cpus));
-	sched_unpin();
+		CPU_OR_ATOMIC(&ipi_nmi_pending, &other_cpus);
 
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
@@ -1373,7 +1360,7 @@ ipi_all_but_self(u_int ipi)
 int
 ipi_nmi_handler()
 {
-	cpuset_t cpumask;
+	u_int cpuid;
 
 	/*
 	 * As long as there is not a simple way to know about a NMI's
@@ -1381,13 +1368,11 @@ ipi_nmi_handler()
 	 * the global pending bitword an IPI_STOP_HARD has been issued
 	 * and should be handled.
 	 */
-	sched_pin();
-	cpumask = PCPU_GET(cpumask);
-	sched_unpin();
-	if (!CPU_OVERLAP(&ipi_nmi_pending, &cpumask))
+	cpuid = PCPU_GET(cpuid);
+	if (!CPU_ISSET(cpuid, &ipi_nmi_pending))
 		return (1);
 
-	CPU_NAND_ATOMIC(&ipi_nmi_pending, &cpumask);
+	CPU_CLR_ATOMIC(cpuid, &ipi_nmi_pending);
 	cpustop_handler();
 	return (0);
 }
@@ -1399,25 +1384,21 @@ ipi_nmi_handler()
 void
 cpustop_handler(void)
 {
-	cpuset_t cpumask;
 	u_int cpu;
 
-	sched_pin();
 	cpu = PCPU_GET(cpuid);
-	cpumask = PCPU_GET(cpumask);
-	sched_unpin();
 
 	savectx(&stoppcbs[cpu]);
 
 	/* Indicate that we are stopped */
-	CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+	CPU_SET_ATOMIC(cpu, &stopped_cpus);
 
 	/* Wait for restart */
-	while (!CPU_OVERLAP(&started_cpus, &cpumask))
+	while (!CPU_ISSET(cpu, &started_cpus))
 	    ia32_pause();
 
-	CPU_NAND_ATOMIC(&started_cpus, &cpumask);
-	CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+	CPU_CLR_ATOMIC(cpu, &started_cpus);
+	CPU_CLR_ATOMIC(cpu, &stopped_cpus);
 
 	if (cpu == 0 && cpustop_restartfunc != NULL) {
 		cpustop_restartfunc();
@@ -1432,19 +1413,17 @@ cpustop_handler(void)
 void
 cpususpend_handler(void)
 {
-	cpuset_t cpumask;
 	register_t cr3, rf;
 	u_int cpu;
 
 	cpu = PCPU_GET(cpuid);
-	cpumask = PCPU_GET(cpumask);
 
 	rf = intr_disable();
 	cr3 = rcr3();
 
 	if (savectx(susppcbs[cpu])) {
 		wbinvd();
-		CPU_OR_ATOMIC(&stopped_cpus, &cpumask);
+		CPU_SET_ATOMIC(cpu, &stopped_cpus);
 	} else {
 		pmap_init_pat();
 		PCPU_SET(switchtime, 0);
@@ -1452,11 +1431,11 @@ cpususpend_handler(void)
 	}
 
 	/* Wait for resume */
-	while (!CPU_OVERLAP(&started_cpus, &cpumask))
+	while (!CPU_ISSET(cpu, &started_cpus))
 		ia32_pause();
 
-	CPU_NAND_ATOMIC(&started_cpus, &cpumask);
-	CPU_NAND_ATOMIC(&stopped_cpus, &cpumask);
+	CPU_CLR_ATOMIC(cpu, &started_cpus);
+	CPU_CLR_ATOMIC(cpu, &stopped_cpus);
 
 	/* Restore CR3 and enable interrupts */
 	load_cr3(cr3);