diff options
| -rw-r--r-- | sys/ia64/ia64/exception.S | 38 | ||||
| -rw-r--r-- | sys/ia64/ia64/pmap.c | 35 |
2 files changed, 67 insertions, 6 deletions
diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S index 4464a88..bbfa738 100644 --- a/sys/ia64/ia64/exception.S +++ b/sys/ia64/ia64/exception.S @@ -170,6 +170,27 @@ ENTRY_NOPROFILE(exception_save, 0) * r30,r31=trapframe pointers * p14,p15=memory stack switch */ + + /* PTC.G enter non-exclusive */ + mov r24 = ar.ccv + movl r25 = pmap_ptc_g_sem + ;; +.ptc_g_0: + ld8.acq r26 = [r25] + ;; + tbit.nz p12, p0 = r26, 63 +(p12) br.cond.spnt.few .ptc_g_0 + ;; + mov ar.ccv = r26 + adds r27 = 1, r26 + ;; + cmpxchg8.rel r27 = [r25], r27, ar.ccv + ;; + cmp.ne p12, p0 = r26, r27 +(p12) br.cond.spnt.few .ptc_g_0 + ;; + mov ar.ccv = r24 + exception_save_restart: { .mmi st8 [r30]=r19,16 // length @@ -407,6 +428,23 @@ exception_save_restart: movl gp=__gp ;; } + + /* PTC.G leave non-exclusive */ + srlz.d + movl r25 = pmap_ptc_g_sem + ;; +.ptc_g_1: + ld8.acq r26 = [r25] + ;; + mov ar.ccv = r26 + adds r27 = -1, r26 + ;; + cmpxchg8.rel r27 = [r25], r27, ar.ccv + ;; + cmp.ne p12, p0 = r26, r27 +(p12) br.cond.spnt.few .ptc_g_1 + ;; + { .mib srlz.d nop 0 diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 523d93d..56a3b49 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -182,7 +182,8 @@ static uint64_t pmap_ptc_e_count1 = 3; static uint64_t pmap_ptc_e_count2 = 2; static uint64_t pmap_ptc_e_stride1 = 0x2000; static uint64_t pmap_ptc_e_stride2 = 0x100000000; -struct mtx pmap_ptcmutex; + +volatile u_long pmap_ptc_g_sem; /* * Data for the RID allocator @@ -340,7 +341,6 @@ pmap_bootstrap() pmap_ptc_e_count2, pmap_ptc_e_stride1, pmap_ptc_e_stride2); - mtx_init(&pmap_ptcmutex, "Global PTC lock", NULL, MTX_SPIN); /* * Setup RIDs. RIDs 0..7 are reserved for the kernel. @@ -540,7 +540,8 @@ pmap_invalidate_page(vm_offset_t va) { struct ia64_lpte *pte; struct pcpu *pc; - uint64_t tag; + uint64_t tag, sem; + register_t is; u_int vhpt_ofs; critical_enter(); @@ -550,10 +551,32 @@ pmap_invalidate_page(vm_offset_t va) pte = (struct ia64_lpte *)(pc->pc_md.vhpt + vhpt_ofs); atomic_cmpset_64(&pte->tag, tag, 1UL << 63); } - critical_exit(); - mtx_lock_spin(&pmap_ptcmutex); + + /* PTC.G enter exclusive */ + is = intr_disable(); + + /* Atomically assert writer after all writers have gone. */ + do { + /* Wait until there's no more writer. */ + do { + sem = atomic_load_acq_long(&pmap_ptc_g_sem); + tag = sem | (1ul << 63); + } while (sem == tag); + } while (!atomic_cmpset_rel_long(&pmap_ptc_g_sem, sem, tag)); + + /* Wait until all readers are gone. */ + tag = (1ul << 63); + do { + sem = atomic_load_acq_long(&pmap_ptc_g_sem); + } while (sem != tag); + ia64_ptc_ga(va, PAGE_SHIFT << 2); - mtx_unlock_spin(&pmap_ptcmutex); + + /* PTC.G leave exclusive */ + atomic_store_rel_long(&pmap_ptc_g_sem, 0); + + intr_restore(is); + critical_exit(); } static void |
