diff options
author | jhb <jhb@FreeBSD.org> | 2006-05-01 21:36:47 +0000 |
---|---|---|
committer | jhb <jhb@FreeBSD.org> | 2006-05-01 21:36:47 +0000 |
commit | ca8d347695197fe6855a628e0325e9ab16820d5f (patch) | |
tree | 3a0d26aaaeb23a38ebdd2566fd5bdbc8c7bde43f /sys | |
parent | 4db7dec298d4cc5de09e6704e4b98919f21bacaf (diff) | |
download | FreeBSD-src-ca8d347695197fe6855a628e0325e9ab16820d5f.zip FreeBSD-src-ca8d347695197fe6855a628e0325e9ab16820d5f.tar.gz |
Add a new 'pmap_invalidate_cache()' to flush the CPU caches via the
wbinvd() instruction. This includes a new IPI so that all CPU caches on
all CPUs are flushed for the SMP case.
MFC after: 1 month
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/amd64/apic_vector.S | 19 | ||||
-rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 8 | ||||
-rw-r--r-- | sys/amd64/amd64/pmap.c | 31 | ||||
-rw-r--r-- | sys/amd64/include/apicvar.h | 5 | ||||
-rw-r--r-- | sys/amd64/include/pmap.h | 1 | ||||
-rw-r--r-- | sys/amd64/include/smp.h | 2 | ||||
-rw-r--r-- | sys/i386/i386/apic_vector.s | 33 | ||||
-rw-r--r-- | sys/i386/i386/mp_machdep.c | 9 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 31 | ||||
-rw-r--r-- | sys/i386/include/apicvar.h | 7 | ||||
-rw-r--r-- | sys/i386/include/pmap.h | 1 | ||||
-rw-r--r-- | sys/i386/include/smp.h | 3 |
12 files changed, 145 insertions, 5 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 96018f3..ab781ca 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -171,6 +171,25 @@ IDTVEC(invlrng) iretq /* + * Invalidate cache. + */ + .text + SUPERALIGN_TEXT +IDTVEC(invlcache) + pushq %rax + + wbinvd + + movq lapic, %rax + movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + + lock + incl smp_tlb_wait + + popq %rax + iretq + +/* * Handler for IPIs sent via the per-cpu IPI bitmap. */ .text diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 226a95c..b2485c6 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -875,6 +875,14 @@ smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offse } void +smp_cache_flush(void) +{ + + if (smp_started) + smp_tlb_shootdown(IPI_INVLCACHE, 0, 0); +} + +void smp_invltlb(void) { diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index fd5e21d..238f0b4 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -732,6 +732,30 @@ pmap_invalidate_all(pmap_t pmap) else critical_exit(); } + +void +pmap_invalidate_cache(void) +{ + + if (smp_started) { + if (!(read_rflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_ipi_mtx); + } else + critical_enter(); + /* + * We need to disable interrupt preemption but MUST NOT have + * interrupts disabled here. + * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again + */ + wbinvd(); + smp_cache_flush(); + if (smp_started) + mtx_unlock_spin(&smp_ipi_mtx); + else + critical_exit(); +} #else /* !SMP */ /* * Normal, non-SMP, invalidation functions. @@ -762,6 +786,13 @@ pmap_invalidate_all(pmap_t pmap) if (pmap == kernel_pmap || pmap->pm_active) invltlb(); } + +PMAP_INLINE void +pmap_invalidate_cache(void) +{ + + wbinvd(); +} #endif /* !SMP */ /* diff --git a/sys/amd64/include/apicvar.h b/sys/amd64/include/apicvar.h index c87dc7e..98cf3a3 100644 --- a/sys/amd64/include/apicvar.h +++ b/sys/amd64/include/apicvar.h @@ -118,8 +118,9 @@ #define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */ #define IPI_INVLPG (APIC_IPI_INTS + 2) #define IPI_INVLRNG (APIC_IPI_INTS + 3) +#define IPI_INVLCACHE (APIC_IPI_INTS + 4) /* Vector to handle bitmap based IPIs */ -#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 5) +#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 6) /* IPIs handled by IPI_BITMAPED_VECTOR (XXX ups is there a better place?) */ #define IPI_AST 0 /* Generate software trap. */ @@ -127,7 +128,7 @@ #define IPI_BITMAP_LAST IPI_PREEMPT #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST) -#define IPI_STOP (APIC_IPI_INTS + 6) /* Stop CPU until restarted. */ +#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */ /* * The spurious interrupt can share the priority class with the IPIs since diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 49e3139..0a774c7 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -309,6 +309,7 @@ void pmap_unmapdev(vm_offset_t, vm_size_t); void pmap_invalidate_page(pmap_t, vm_offset_t); void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t); void pmap_invalidate_all(pmap_t); +void pmap_invalidate_cache(void); #endif /* _KERNEL */ diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 94a7022..5a2d3aa 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -40,6 +40,7 @@ inthand_t IDTVEC(invltlb), /* TLB shootdowns - global */ IDTVEC(invlpg), /* TLB shootdowns - 1 page */ IDTVEC(invlrng), /* TLB shootdowns - page range */ + IDTVEC(invlcache), /* Write back and invalidate cache */ IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ IDTVEC(cpustop), /* CPU stops & waits to be restarted */ IDTVEC(rendezvous); /* handle CPU rendezvous */ @@ -56,6 +57,7 @@ void ipi_bitmap_handler(struct trapframe frame); u_int mp_bootaddress(u_int); int mp_grab_cpu_hlt(void); void mp_topology(void); +void smp_cache_flush(void); void smp_invlpg(vm_offset_t addr); void smp_masked_invlpg(u_int mask, vm_offset_t addr); void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva); diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index a9d8af7..61dec19 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -233,6 +233,39 @@ IDTVEC(invlrng) iret /* + * Invalidate cache. + */ + .text + SUPERALIGN_TEXT +IDTVEC(invlcache) + pushl %eax + pushl %ds + movl $KDSEL, %eax /* Kernel data selector */ + movl %eax, %ds + +#ifdef COUNT_IPIS + pushl %fs + movl $KPSEL, %eax /* Private space selector */ + movl %eax, %fs + movl PCPU(CPUID), %eax + popl %fs + movl ipi_invlcache_counts(,%eax,4),%eax + incl (%eax) +#endif + + wbinvd + + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ + + lock + incl smp_tlb_wait + + popl %ds + popl %eax + iret + +/* * Handler for IPIs sent via the per-cpu IPI bitmap. */ .text diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 3bfce36..5f468c0 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -171,6 +171,7 @@ static u_long *ipi_ast_counts[MAXCPU]; u_long *ipi_invltlb_counts[MAXCPU]; u_long *ipi_invlrng_counts[MAXCPU]; u_long *ipi_invlpg_counts[MAXCPU]; +u_long *ipi_invlcache_counts[MAXCPU]; u_long *ipi_rendezvous_counts[MAXCPU]; u_long *ipi_lazypmap_counts[MAXCPU]; #endif @@ -1047,6 +1048,14 @@ smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offse } void +smp_cache_flush(void) +{ + + if (smp_started) + smp_tlb_shootdown(IPI_INVLCACHE, 0, 0); +} + +void smp_invltlb(void) { diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index f09b6bf..d28bf52 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -698,6 +698,30 @@ pmap_invalidate_all(pmap_t pmap) else critical_exit(); } + +void +pmap_invalidate_cache(void) +{ + + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_ipi_mtx); + } else + critical_enter(); + /* + * We need to disable interrupt preemption but MUST NOT have + * interrupts disabled here. + * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again + */ + wbinvd(); + smp_cache_flush(); + if (smp_started) + mtx_unlock_spin(&smp_ipi_mtx); + else + critical_exit(); +} #else /* !SMP */ /* * Normal, non-SMP, 486+ invalidation functions. @@ -728,6 +752,13 @@ pmap_invalidate_all(pmap_t pmap) if (pmap == kernel_pmap || pmap->pm_active) invltlb(); } + +PMAP_INLINE void +pmap_invalidate_cache(void) +{ + + wbinvd(); +} #endif /* !SMP */ /* diff --git a/sys/i386/include/apicvar.h b/sys/i386/include/apicvar.h index 8d1a6be..bca7bb1 100644 --- a/sys/i386/include/apicvar.h +++ b/sys/i386/include/apicvar.h @@ -116,9 +116,10 @@ #define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */ #define IPI_INVLPG (APIC_IPI_INTS + 2) #define IPI_INVLRNG (APIC_IPI_INTS + 3) -#define IPI_LAZYPMAP (APIC_IPI_INTS + 4) /* Lazy pmap release. */ +#define IPI_INVLCACHE (APIC_IPI_INTS + 4) +#define IPI_LAZYPMAP (APIC_IPI_INTS + 5) /* Lazy pmap release. */ /* Vector to handle bitmap based IPIs */ -#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 5) +#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 6) /* IPIs handled by IPI_BITMAPED_VECTOR (XXX ups is there a better place?) */ #define IPI_AST 0 /* Generate software trap. */ @@ -126,7 +127,7 @@ #define IPI_BITMAP_LAST IPI_PREEMPT #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST) -#define IPI_STOP (APIC_IPI_INTS + 6) /* Stop CPU until restarted. */ +#define IPI_STOP (APIC_IPI_INTS + 7) /* Stop CPU until restarted. */ /* * The spurious interrupt can share the priority class with the IPIs since diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h index 5f9424d..b60d439 100644 --- a/sys/i386/include/pmap.h +++ b/sys/i386/include/pmap.h @@ -378,6 +378,7 @@ void pmap_set_pg(void); void pmap_invalidate_page(pmap_t, vm_offset_t); void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t); void pmap_invalidate_all(pmap_t); +void pmap_invalidate_cache(void); #endif /* _KERNEL */ diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h index bd67a33..6451a5c 100644 --- a/sys/i386/include/smp.h +++ b/sys/i386/include/smp.h @@ -39,6 +39,7 @@ extern struct mtx smp_tlb_mtx; extern u_long *ipi_invltlb_counts[MAXCPU]; extern u_long *ipi_invlrng_counts[MAXCPU]; extern u_long *ipi_invlpg_counts[MAXCPU]; +extern u_long *ipi_invlcache_counts[MAXCPU]; extern u_long *ipi_rendezvous_counts[MAXCPU]; extern u_long *ipi_lazypmap_counts[MAXCPU]; #endif @@ -48,6 +49,7 @@ inthand_t IDTVEC(invltlb), /* TLB shootdowns - global */ IDTVEC(invlpg), /* TLB shootdowns - 1 page */ IDTVEC(invlrng), /* TLB shootdowns - page range */ + IDTVEC(invlcache), /* Write back and invalidate cache */ IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ IDTVEC(cpustop), /* CPU stops & waits to be restarted */ IDTVEC(rendezvous), /* handle CPU rendezvous */ @@ -65,6 +67,7 @@ void ipi_bitmap_handler(struct trapframe frame); u_int mp_bootaddress(u_int); int mp_grab_cpu_hlt(void); void mp_topology(void); +void smp_cache_flush(void); void smp_invlpg(vm_offset_t addr); void smp_masked_invlpg(u_int mask, vm_offset_t addr); void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva); |