diff options
author | mav <mav@FreeBSD.org> | 2010-06-17 11:54:49 +0000 |
---|---|---|
committer | mav <mav@FreeBSD.org> | 2010-06-17 11:54:49 +0000 |
commit | 71d7c38373261bf5664d7f1a1890eef6c2f30356 (patch) | |
tree | cbf8d07299f47472d94b91ec634c2217df714910 /sys | |
parent | 9f2d4c3357718975468ee4d1f2479c9dd704adfd (diff) | |
download | FreeBSD-src-71d7c38373261bf5664d7f1a1890eef6c2f30356.zip FreeBSD-src-71d7c38373261bf5664d7f1a1890eef6c2f30356.tar.gz |
Merge COUNT_XINVLTLB_HITS and COUNT_IPIS kernel options from i386 to amd64.
This information can be very valuable for CPU sleep-time (and respectively
idle power consumption) optimization.
Add counters for timer-related IPIs.
Reviewed by: jhb@ (previous version)
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/amd64/apic_vector.S | 54 | ||||
-rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 137 | ||||
-rw-r--r-- | sys/amd64/conf/NOTES | 5 | ||||
-rw-r--r-- | sys/amd64/include/smp.h | 8 | ||||
-rw-r--r-- | sys/conf/options.amd64 | 2 | ||||
-rw-r--r-- | sys/i386/i386/mp_machdep.c | 35 |
6 files changed, 221 insertions, 20 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 1c044b8..73a0eab 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -36,6 +36,8 @@ * as well as IPI handlers. */ +#include "opt_smp.h" + #include <machine/asmacros.h> #include <machine/apicreg.h> @@ -135,6 +137,19 @@ IDTVEC(errorint) .text SUPERALIGN_TEXT IDTVEC(invltlb) +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + PUSH_FRAME + movl PCPU(CPUID), %eax +#ifdef COUNT_XINVLTLB_HITS + incl xhits_gbl(,%rax,4) +#endif +#ifdef COUNT_IPIS + movq ipi_invltlb_counts(,%rax,8),%rax + incq (%rax) +#endif + POP_FRAME +#endif + pushq %rax movq %cr3, %rax /* invalidate the TLB */ @@ -155,6 +170,19 @@ IDTVEC(invltlb) .text SUPERALIGN_TEXT IDTVEC(invlpg) +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + PUSH_FRAME + movl PCPU(CPUID), %eax +#ifdef COUNT_XINVLTLB_HITS + incl xhits_pg(,%rax,4) +#endif +#ifdef COUNT_IPIS + movq ipi_invlpg_counts(,%rax,8),%rax + incq (%rax) +#endif + POP_FRAME +#endif + pushq %rax movq smp_tlb_addr1, %rax @@ -175,6 +203,19 @@ IDTVEC(invlpg) .text SUPERALIGN_TEXT IDTVEC(invlrng) +#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS) + PUSH_FRAME + movl PCPU(CPUID), %eax +#ifdef COUNT_XINVLTLB_HITS + incl xhits_rng(,%rax,4) +#endif +#ifdef COUNT_IPIS + movq ipi_invlrng_counts(,%rax,8),%rax + incq (%rax) +#endif + POP_FRAME +#endif + pushq %rax pushq %rdx @@ -201,6 +242,14 @@ IDTVEC(invlrng) .text SUPERALIGN_TEXT IDTVEC(invlcache) +#ifdef COUNT_IPIS + PUSH_FRAME + movl PCPU(CPUID), %eax + movq ipi_invlcache_counts(,%rax,8),%rax + incq (%rax) + POP_FRAME +#endif + pushq %rax wbinvd @@ -270,6 +319,11 @@ IDTVEC(cpususpend) SUPERALIGN_TEXT IDTVEC(rendezvous) PUSH_FRAME +#ifdef COUNT_IPIS + movl PCPU(CPUID), %eax + movq ipi_rendezvous_counts(,%rax,8), %rax + incq (%rax) +#endif call smp_rendezvous_action movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 1bd7200..0de3549 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include "opt_mp_watchdog.h" #include "opt_sched.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -106,6 +107,20 @@ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; +#ifdef COUNT_IPIS +/* Interrupt counts. */ +static u_long *ipi_preempt_counts[MAXCPU]; +static u_long *ipi_ast_counts[MAXCPU]; +u_long *ipi_invltlb_counts[MAXCPU]; +u_long *ipi_invlrng_counts[MAXCPU]; +u_long *ipi_invlpg_counts[MAXCPU]; +u_long *ipi_invlcache_counts[MAXCPU]; +u_long *ipi_rendezvous_counts[MAXCPU]; +u_long *ipi_lazypmap_counts[MAXCPU]; +static u_long *ipi_hardclock_counts[MAXCPU]; +static u_long *ipi_statclock_counts[MAXCPU]; +#endif + extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); /* @@ -970,6 +985,42 @@ start_ap(int apic_id) return 0; /* return FAILURE */ } +#ifdef COUNT_XINVLTLB_HITS +u_int xhits_gbl[MAXCPU]; +u_int xhits_pg[MAXCPU]; +u_int xhits_rng[MAXCPU]; +SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, + sizeof(xhits_gbl), "IU", ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, + sizeof(xhits_pg), "IU", ""); +SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, + sizeof(xhits_rng), "IU", ""); + +u_int ipi_global; +u_int ipi_page; +u_int ipi_range; +u_int ipi_range_size; +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, + 0, ""); + +u_int ipi_masked_global; +u_int ipi_masked_page; +u_int ipi_masked_range; +u_int ipi_masked_range_size; +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, + &ipi_masked_global, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, + &ipi_masked_page, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, + &ipi_masked_range, 0, ""); +SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, + &ipi_masked_range_size, 0, ""); +#endif /* COUNT_XINVLTLB_HITS */ + /* * Flush the TLB on all other CPU's */ @@ -1047,6 +1098,9 @@ smp_invltlb(void) if (smp_started) { smp_tlb_shootdown(IPI_INVLTLB, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_global++; +#endif } } @@ -1054,8 +1108,12 @@ void smp_invlpg(vm_offset_t addr) { - if (smp_started) + if (smp_started) { smp_tlb_shootdown(IPI_INVLPG, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_page++; +#endif + } } void @@ -1064,6 +1122,10 @@ smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) if (smp_started) { smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_range++; + ipi_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif } } @@ -1073,6 +1135,9 @@ smp_masked_invltlb(cpumask_t mask) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_global++; +#endif } } @@ -1082,6 +1147,9 @@ smp_masked_invlpg(cpumask_t mask, vm_offset_t addr) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_page++; +#endif } } @@ -1091,6 +1159,10 @@ smp_masked_invlpg_range(cpumask_t mask, vm_offset_t addr1, vm_offset_t addr2) if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_masked_range++; + ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif } } @@ -1102,16 +1174,30 @@ ipi_bitmap_handler(struct trapframe frame) ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]); - if (ipi_bitmap & (1 << IPI_PREEMPT)) + if (ipi_bitmap & (1 << IPI_PREEMPT)) { +#ifdef COUNT_IPIS + (*ipi_preempt_counts[cpu])++; +#endif sched_preempt(curthread); - - /* Nothing to do for AST */ - - if (ipi_bitmap & (1 << IPI_HARDCLOCK)) + } + if (ipi_bitmap & (1 << IPI_AST)) { +#ifdef COUNT_IPIS + (*ipi_ast_counts[cpu])++; +#endif + /* Nothing to do for AST */ + } + if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_hardclock_counts[cpu])++; +#endif hardclockintr(&frame); - - if (ipi_bitmap & (1 << IPI_STATCLOCK)) + } + if (ipi_bitmap & (1 << IPI_STATCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_statclock_counts[cpu])++; +#endif statclockintr(&frame); + } } /* @@ -1432,3 +1518,38 @@ mp_grab_cpu_hlt(void) __asm __volatile("sti; hlt" : : : "memory"); return (retval); } + +#ifdef COUNT_IPIS +/* + * Setup interrupt counters for IPI handlers. + */ +static void +mp_ipi_intrcnt(void *dummy) +{ + char buf[64]; + int i; + + CPU_FOREACH(i) { + snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); + intrcnt_add(buf, &ipi_invltlb_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); + intrcnt_add(buf, &ipi_invlrng_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); + intrcnt_add(buf, &ipi_invlpg_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:preempt", i); + intrcnt_add(buf, &ipi_preempt_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:ast", i); + intrcnt_add(buf, &ipi_ast_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); + intrcnt_add(buf, &ipi_rendezvous_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:lazypmap", i); + intrcnt_add(buf, &ipi_lazypmap_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); + intrcnt_add(buf, &ipi_hardclock_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:statclock", i); + intrcnt_add(buf, &ipi_statclock_counts[i]); + } +} +SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); +#endif + diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES index dbd689c..b16db62 100644 --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -30,6 +30,11 @@ device mptable # Optional MPSPEC mptable support # options MP_WATCHDOG +# Debugging options. +# +options COUNT_XINVLTLB_HITS # Counters for TLB events +options COUNT_IPIS # Per-CPU IPI interrupt counters + ##################################################################### diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 1cc21a4..feebe92 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -35,6 +35,14 @@ extern int mp_naps; extern int boot_cpu_id; extern struct pcb stoppcbs[]; extern int cpu_apic_ids[]; +#ifdef COUNT_IPIS +extern u_long *ipi_invltlb_counts[MAXCPU]; +extern u_long *ipi_invlrng_counts[MAXCPU]; +extern u_long *ipi_invlpg_counts[MAXCPU]; +extern u_long *ipi_invlcache_counts[MAXCPU]; +extern u_long *ipi_rendezvous_counts[MAXCPU]; +extern u_long *ipi_lazypmap_counts[MAXCPU]; +#endif /* IPI handlers */ inthand_t diff --git a/sys/conf/options.amd64 b/sys/conf/options.amd64 index 20a49a1..78660e2 100644 --- a/sys/conf/options.amd64 +++ b/sys/conf/options.amd64 @@ -3,6 +3,8 @@ AUTO_EOI_1 opt_auto_eoi.h AUTO_EOI_2 opt_auto_eoi.h +COUNT_XINVLTLB_HITS opt_smp.h +COUNT_IPIS opt_smp.h MAXMEM PERFMON PMAP_SHPGPERPROC opt_pmap.h diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 36d5492..70f69271 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -166,6 +166,8 @@ u_long *ipi_invlpg_counts[MAXCPU]; u_long *ipi_invlcache_counts[MAXCPU]; u_long *ipi_rendezvous_counts[MAXCPU]; u_long *ipi_lazypmap_counts[MAXCPU]; +static u_long *ipi_hardclock_counts[MAXCPU]; +static u_long *ipi_statclock_counts[MAXCPU]; #endif /* @@ -1266,19 +1268,24 @@ ipi_bitmap_handler(struct trapframe frame) #endif sched_preempt(curthread); } - if (ipi_bitmap & (1 << IPI_AST)) { #ifdef COUNT_IPIS (*ipi_ast_counts[cpu])++; #endif /* Nothing to do for AST */ } - - if (ipi_bitmap & (1 << IPI_HARDCLOCK)) + if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_hardclock_counts[cpu])++; +#endif hardclockintr(&frame); - - if (ipi_bitmap & (1 << IPI_STATCLOCK)) + } + if (ipi_bitmap & (1 << IPI_STATCLOCK)) { +#ifdef COUNT_IPIS + (*ipi_statclock_counts[cpu])++; +#endif statclockintr(&frame); + } } /* @@ -1574,20 +1581,24 @@ mp_ipi_intrcnt(void *dummy) int i; CPU_FOREACH(i) { - snprintf(buf, sizeof(buf), "cpu%d: invltlb", i); + snprintf(buf, sizeof(buf), "cpu%d:invltlb", i); intrcnt_add(buf, &ipi_invltlb_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: invlrng", i); + snprintf(buf, sizeof(buf), "cpu%d:invlrng", i); intrcnt_add(buf, &ipi_invlrng_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: invlpg", i); + snprintf(buf, sizeof(buf), "cpu%d:invlpg", i); intrcnt_add(buf, &ipi_invlpg_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: preempt", i); + snprintf(buf, sizeof(buf), "cpu%d:preempt", i); intrcnt_add(buf, &ipi_preempt_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: ast", i); + snprintf(buf, sizeof(buf), "cpu%d:ast", i); intrcnt_add(buf, &ipi_ast_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: rendezvous", i); + snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); intrcnt_add(buf, &ipi_rendezvous_counts[i]); - snprintf(buf, sizeof(buf), "cpu%d: lazypmap", i); + snprintf(buf, sizeof(buf), "cpu%d:lazypmap", i); intrcnt_add(buf, &ipi_lazypmap_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); + intrcnt_add(buf, &ipi_hardclock_counts[i]); + snprintf(buf, sizeof(buf), "cpu%d:statclock", i); + intrcnt_add(buf, &ipi_statclock_counts[i]); } } SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); |