diff options
Diffstat (limited to 'sys/i386/include')
| -rw-r--r-- | sys/i386/include/cpufunc.h | 244 | ||||
| -rw-r--r-- | sys/i386/include/mptable.h | 211 | ||||
| -rw-r--r-- | sys/i386/include/pmap.h | 4 | ||||
| -rw-r--r-- | sys/i386/include/smp.h | 3 | ||||
| -rw-r--r-- | sys/i386/include/smptests.h | 7 |
5 files changed, 104 insertions, 365 deletions
diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index 94d5c3a..969541f 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -227,6 +227,62 @@ invd(void) __asm __volatile("invd"); } +#if defined(SMP) && defined(_KERNEL) + +/* + * When using APIC IPI's, invlpg() is not simply the invlpg instruction + * (this is a bug) and the inlining cost is prohibitive since the call + * executes into the IPI transmission system. + */ +void invlpg __P((u_int addr)); +void invltlb __P((void)); + +static __inline void +cpu_invlpg(void *addr) +{ + __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); +} + +static __inline void +cpu_invltlb(void) +{ + u_int temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() + * is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) + : : "memory"); +#if defined(SWTCH_OPTIM_STATS) + ++tlb_flush_count; +#endif +} + +#else /* !(SMP && _KERNEL) */ + +static __inline void +invlpg(u_int addr) +{ + __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); +} + +static __inline void +invltlb(void) +{ + u_int temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() + * is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) + : : "memory"); +#ifdef SWTCH_OPTIM_STATS + ++tlb_flush_count; +#endif +} + +#endif /* SMP && _KERNEL */ + static __inline u_short inw(u_int port) { @@ -292,6 +348,15 @@ outw(u_int port, u_short data) } static __inline u_int +rcr2(void) +{ + u_int data; + + __asm __volatile("movl %%cr2,%0" : "=r" (data)); + return (data); +} + +static __inline u_int read_eflags(void) { u_int ef; @@ -355,162 +420,6 @@ wrmsr(u_int msr, u_int64_t newval) __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); } -static __inline void -load_cr0(u_int data) -{ - - __asm __volatile("movl %0,%%cr0" : : "r" (data)); -} - -static __inline u_int -rcr0(void) -{ - u_int data; - - __asm __volatile("movl %%cr0,%0" : "=r" (data)); - return (data); -} - -static __inline u_int -rcr2(void) -{ - u_int data; - - __asm __volatile("movl %%cr2,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_cr3(u_int data) -{ - - __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); -#if defined(SWTCH_OPTIM_STATS) - ++tlb_flush_count; -#endif -} - -static __inline u_int -rcr3(void) -{ - u_int data; - - __asm __volatile("movl %%cr3,%0" : "=r" (data)); - return (data); -} - -static __inline void -load_cr4(u_int data) -{ - __asm __volatile("movl %0,%%cr4" : : "r" (data)); -} - -static __inline u_int -rcr4(void) -{ - u_int data; - - __asm __volatile("movl %%cr4,%0" : "=r" (data)); - return (data); -} - -/* - * Global TLB flush (except for thise for pages marked PG_G) - */ -static __inline void -cpu_invltlb(void) -{ - - load_cr3(rcr3()); -} - -/* - * TLB flush for an individual page (even if it has PG_G). - * Only works on 486+ CPUs (i386 does not have PG_G). - */ -static __inline void -cpu_invlpg(u_int addr) -{ - -#ifndef I386_CPU - __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); -#else - cpu_invltlb(); -#endif -} - -#ifdef PAGE_SIZE /* Avoid this file depending on sys/param.h */ -/* - * Same as above but for a range of pages. - */ -static __inline void -cpu_invlpg_range(u_int startva, u_int endva) -{ -#ifndef I386_CPU - u_int addr; - - for (addr = startva; addr < endva; addr += PAGE_SIZE) - __asm __volatile("invlpg %0" : : "m" (*(char *)addr)); - __asm __volatile("" : : : "memory"); -#else - cpu_invltlb(); -#endif -} -#endif - -#ifdef SMP -extern void smp_invlpg(u_int addr); -extern void smp_masked_invlpg(u_int mask, u_int addr); -#ifdef PAGE_SIZE /* Avoid this file depending on sys/param.h */ -extern void smp_invlpg_range(u_int startva, u_int endva); -extern void smp_masked_invlpg_range(u_int mask, u_int startva, u_int endva); -#endif -extern void smp_invltlb(void); -extern void smp_masked_invltlb(u_int mask); -#endif - -/* - * Generic page TLB flush. Takes care of SMP. - */ -static __inline void -invlpg(u_int addr) -{ - - cpu_invlpg(addr); -#ifdef SMP - smp_invlpg(addr); -#endif -} - -#ifdef PAGE_SIZE /* Avoid this file depending on sys/param.h */ -/* - * Generic TLB flush for a range of pages. Takes care of SMP. - * Saves many IPIs for SMP mode. - */ -static __inline void -invlpg_range(u_int startva, u_int endva) -{ - - cpu_invlpg_range(startva, endva); -#ifdef SMP - smp_invlpg_range(startva, endva); -#endif -} -#endif - -/* - * Generic global TLB flush (except for thise for pages marked PG_G) - */ -static __inline void -invltlb(void) -{ - - cpu_invltlb(); -#ifdef SMP - smp_invltlb(); -#endif -} - static __inline u_int rfs(void) { @@ -672,8 +581,6 @@ cpu_critical_exit(critical_t eflags) int breakpoint __P((void)); u_int bsfl __P((u_int mask)); u_int bsrl __P((u_int mask)); -void cpu_invlpg __P((u_int addr)); -void cpu_invlpg_range __P((u_int start, u_int end)); void disable_intr __P((void)); void do_cpuid __P((u_int ax, u_int *p)); void enable_intr __P((void)); @@ -684,26 +591,15 @@ void insl __P((u_int port, void *addr, size_t cnt)); void insw __P((u_int port, void *addr, size_t cnt)); void invd __P((void)); void invlpg __P((u_int addr)); -void invlpg_range __P((u_int start, u_int end)); void invltlb __P((void)); u_short inw __P((u_int port)); -void load_cr0 __P((u_int cr0)); -void load_cr3 __P((u_int cr3)); -void load_cr4 __P((u_int cr4)); -void load_fs __P((u_int sel)); -void load_gs __P((u_int sel)); void outb __P((u_int port, u_char data)); void outl __P((u_int port, u_int data)); void outsb __P((u_int port, void *addr, size_t cnt)); void outsl __P((u_int port, void *addr, size_t cnt)); void outsw __P((u_int port, void *addr, size_t cnt)); void outw __P((u_int port, u_short data)); -u_int rcr0 __P((void)); u_int rcr2 __P((void)); -u_int rcr3 __P((void)); -u_int rcr4 __P((void)); -u_int rfs __P((void)); -u_int rgs __P((void)); u_int64_t rdmsr __P((u_int msr)); u_int64_t rdpmc __P((u_int pmc)); u_int64_t rdtsc __P((void)); @@ -711,12 +607,22 @@ u_int read_eflags __P((void)); void wbinvd __P((void)); void write_eflags __P((u_int ef)); void wrmsr __P((u_int msr, u_int64_t newval)); +u_int rfs __P((void)); +u_int rgs __P((void)); +void load_fs __P((u_int sel)); +void load_gs __P((u_int sel)); critical_t cpu_critical_enter __P((void)); void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ +void load_cr0 __P((u_int cr0)); +void load_cr3 __P((u_int cr3)); +void load_cr4 __P((u_int cr4)); void ltr __P((u_short sel)); +u_int rcr0 __P((void)); +u_int rcr3 __P((void)); +u_int rcr4 __P((void)); void reset_dbregs __P((void)); __END_DECLS diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index 008dfc5..27ee7ae 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -287,14 +287,6 @@ extern pt_entry_t *SMPpt; struct pcb stoppcbs[MAXCPU]; -#ifdef APIC_IO -/* Variables needed for SMP tlb shootdown. */ -u_int smp_tlb_addr1; -u_int smp_tlb_addr2; -volatile int smp_tlb_wait; -static struct mtx smp_tlb_mtx; -#endif - /* * Local data and functions. */ @@ -343,9 +335,6 @@ init_locks(void) #ifdef USE_COMLOCK mtx_init(&com_mtx, "com", MTX_SPIN); #endif /* USE_COMLOCK */ -#ifdef APIC_IO - mtx_init(&smp_tlb_mtx, "tlb", MTX_SPIN); -#endif } /* @@ -615,10 +604,6 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - setidt(XINVLPG_OFFSET, Xinvlpg, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - setidt(XINVLRNG_OFFSET, Xinvlrng, - SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for forwarding hardclock() */ setidt(XHARDCLOCK_OFFSET, Xhardclock, @@ -2201,198 +2186,42 @@ start_ap(int logical_cpu, u_int boot_addr) return 0; /* return FAILURE */ } -#if defined(APIC_IO) - -#ifdef COUNT_XINVLTLB_HITS -u_int xhits_gbl[MAXCPU]; -u_int xhits_pg[MAXCPU]; -u_int xhits_rng[MAXCPU]; -SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); -SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, - sizeof(xhits_gbl), "IU", ""); -SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, - sizeof(xhits_pg), "IU", ""); -SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, - sizeof(xhits_rng), "IU", ""); - -u_int ipi_global; -u_int ipi_page; -u_int ipi_range; -u_int ipi_range_size; -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, - 0, ""); - -u_int ipi_masked_global; -u_int ipi_masked_page; -u_int ipi_masked_range; -u_int ipi_masked_range_size; -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, - &ipi_masked_global, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, - &ipi_masked_page, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, - &ipi_masked_range, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, - &ipi_masked_range_size, 0, ""); -#endif - /* * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. */ -static void -smp_tlb_shootdown(u_int vector, u_int addr1, u_int addr2) -{ - u_int ncpu; - register_t eflags; - - ncpu = mp_ncpus - 1; /* does not shootdown self */ - if (ncpu < 1) - return; /* no other cpus */ - eflags = read_eflags(); - if ((eflags & PSL_I) == 0) - panic("absolutely cannot call smp_ipi_shootdown with interrupts already disabled"); - mtx_lock_spin(&smp_tlb_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - smp_tlb_wait = 0; - ipi_all_but_self(vector); - while (atomic_load_acq_int(&smp_tlb_wait) < ncpu) - /* XXX cpu_pause() */ ; - mtx_unlock_spin(&smp_tlb_mtx); -} - -static void -smp_targeted_tlb_shootdown(u_int mask, u_int vector, u_int addr1, u_int addr2) -{ - u_int m; - int i, ncpu, othercpus; - register_t eflags; - - othercpus = mp_ncpus - 1; - if (mask == (u_int)-1) { - ncpu = othercpus; - if (ncpu < 1) - return; - } else { - /* XXX there should be a pcpu self mask */ - mask &= ~(1 << PCPU_GET(cpuid)); - if (mask == 0) - return; - /* Count the target cpus */ - ncpu = 0; - m = mask; - while ((i = ffs(m)) != 0) { - m >>= i; - ncpu++; - } - if (ncpu > othercpus) { - /* XXX this should be a panic offence */ - printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", - ncpu, othercpus); - ncpu = othercpus; - } - /* XXX should be a panic, implied by mask == 0 above */ - if (ncpu < 1) - return; - } - eflags = read_eflags(); - if ((eflags & PSL_I) == 0) - panic("absolutely cannot call smp_targeted_ipi_shootdown with interrupts already disabled"); - mtx_lock_spin(&smp_tlb_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - smp_tlb_wait = 0; - if (mask == (u_int)-1) - ipi_all_but_self(vector); - else - ipi_selected(mask, vector); - while (atomic_load_acq_int(&smp_tlb_wait) < ncpu) - /* XXX cpu_pause() */ ; - mtx_unlock_spin(&smp_tlb_mtx); -} -#endif - void smp_invltlb(void) { #if defined(APIC_IO) - if (smp_started) { - smp_tlb_shootdown(IPI_INVLTLB, 0, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_global++; -#endif - } + if (smp_started) + ipi_all_but_self(IPI_INVLTLB); #endif /* APIC_IO */ } void -smp_invlpg(u_int addr) +invlpg(u_int addr) { -#if defined(APIC_IO) - if (smp_started) { - smp_tlb_shootdown(IPI_INVLPG, addr, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_page++; -#endif - } -#endif /* APIC_IO */ -} + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); -void -smp_invlpg_range(u_int addr1, u_int addr2) -{ -#if defined(APIC_IO) - if (smp_started) { - smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); -#ifdef COUNT_XINVLTLB_HITS - ipi_range++; - ipi_range_size += (addr2 - addr1) / PAGE_SIZE; -#endif - } -#endif /* APIC_IO */ + /* send a message to the other CPUs */ + smp_invltlb(); } void -smp_masked_invltlb(u_int mask) +invltlb(void) { -#if defined(APIC_IO) - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_masked_global++; -#endif - } -#endif /* APIC_IO */ -} + u_long temp; -void -smp_masked_invlpg(u_int mask, u_int addr) -{ -#if defined(APIC_IO) - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_masked_page++; -#endif - } -#endif /* APIC_IO */ -} + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); -void -smp_masked_invlpg_range(u_int mask, u_int addr1, u_int addr2) -{ -#if defined(APIC_IO) - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); -#ifdef COUNT_XINVLTLB_HITS - ipi_masked_range++; - ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; -#endif - } -#endif /* APIC_IO */ + /* send a message to the other CPUs */ + smp_invltlb(); } @@ -2451,9 +2280,6 @@ ap_init(void) /* Build our map of 'other' CPUs. */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); - if (bootverbose) - apic_dump("ap_init()"); - printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); if (smp_cpus == mp_ncpus) { @@ -2486,8 +2312,7 @@ forwarded_statclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - statclock_process(curthread->td_kse, TRAPF_PC(&frame), - TRAPF_USERMODE(&frame)); + statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h index 618bb3f..7358a9e 100644 --- a/sys/i386/include/pmap.h +++ b/sys/i386/include/pmap.h @@ -151,7 +151,7 @@ extern pt_entry_t PTmap[], APTmap[]; extern pd_entry_t PTD[], APTD[]; extern pd_entry_t PTDpde, APTDpde; -extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */ +extern pd_entry_t IdlePTD; /* physical address of "Idle" state directory */ #endif #ifdef _KERNEL @@ -267,7 +267,9 @@ void *pmap_mapdev __P((vm_offset_t, vm_size_t)); void pmap_unmapdev __P((vm_offset_t, vm_size_t)); pt_entry_t *pmap_pte __P((pmap_t, vm_offset_t)) __pure2; vm_page_t pmap_use_pt __P((pmap_t, vm_offset_t)); +#ifdef SMP void pmap_set_opt __P((void)); +#endif #endif /* _KERNEL */ diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h index 4136c20..34228e2 100644 --- a/sys/i386/include/smp.h +++ b/sys/i386/include/smp.h @@ -51,8 +51,6 @@ extern int current_postcode; /** XXX currently in mp_machdep.c */ * Interprocessor interrupts for SMP. */ #define IPI_INVLTLB XINVLTLB_OFFSET -#define IPI_INVLPG XINVLPG_OFFSET -#define IPI_INVLRNG XINVLRNG_OFFSET #define IPI_RENDEZVOUS XRENDEZVOUS_OFFSET #define IPI_AST XCPUAST_OFFSET #define IPI_STOP XCPUSTOP_OFFSET @@ -109,6 +107,7 @@ void assign_apic_irq __P((int apic, int intpin, int irq)); void revoke_apic_irq __P((int irq)); void bsp_apic_configure __P((void)); void init_secondary __P((void)); +void smp_invltlb __P((void)); void forward_statclock __P((void)); void forwarded_statclock __P((struct trapframe frame)); void forward_hardclock __P((void)); diff --git a/sys/i386/include/smptests.h b/sys/i386/include/smptests.h index ea8e84b..d666148 100644 --- a/sys/i386/include/smptests.h +++ b/sys/i386/include/smptests.h @@ -90,6 +90,13 @@ */ #define APIC_INTR_REORDER +/* + * Redirect clock interrupts to a higher priority (fast intr) vector, + * while still using the slow interrupt handler. Only effective when + * APIC_INTR_REORDER is defined. + */ +#define APIC_INTR_HIGHPRI_CLOCK + #endif /* APIC_IO */ /* |
