diff options
Diffstat (limited to 'arch/parisc/include')
-rw-r--r-- | arch/parisc/include/asm/atomic.h | 63 | ||||
-rw-r--r-- | arch/parisc/include/asm/hash.h | 146 | ||||
-rw-r--r-- | arch/parisc/include/asm/pgalloc.h | 7 | ||||
-rw-r--r-- | arch/parisc/include/asm/spinlock.h | 9 | ||||
-rw-r--r-- | arch/parisc/include/asm/traps.h | 2 |
5 files changed, 213 insertions, 14 deletions
diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 1d10999..5394b9c 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -121,16 +121,39 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ return ret; \ } -#define ATOMIC_OPS(op, c_op) ATOMIC_OP(op, c_op) ATOMIC_OP_RETURN(op, c_op) +#define ATOMIC_FETCH_OP(op, c_op) \ +static __inline__ int atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + int ret; \ + \ + _atomic_spin_lock_irqsave(v, flags); \ + ret = v->counter; \ + v->counter c_op i; \ + _atomic_spin_unlock_irqrestore(v, flags); \ + \ + return ret; \ +} + +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_OP_RETURN(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op, c_op) \ + ATOMIC_OP(op, c_op) \ + ATOMIC_FETCH_OP(op, c_op) + +ATOMIC_OPS(and, &=) +ATOMIC_OPS(or, |=) +ATOMIC_OPS(xor, ^=) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -185,15 +208,39 @@ static __inline__ s64 atomic64_##op##_return(s64 i, atomic64_t *v) \ return ret; \ } -#define ATOMIC64_OPS(op, c_op) ATOMIC64_OP(op, c_op) ATOMIC64_OP_RETURN(op, c_op) +#define ATOMIC64_FETCH_OP(op, c_op) \ +static __inline__ s64 atomic64_fetch_##op(s64 i, atomic64_t *v) \ +{ \ + unsigned long flags; \ + s64 ret; \ + \ + _atomic_spin_lock_irqsave(v, flags); \ + ret = v->counter; \ + v->counter c_op i; \ + _atomic_spin_unlock_irqrestore(v, flags); \ + \ + return ret; \ +} + +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op, c_op) \ + ATOMIC64_OP_RETURN(op, c_op) \ + ATOMIC64_FETCH_OP(op, c_op) ATOMIC64_OPS(add, +=) ATOMIC64_OPS(sub, -=) -ATOMIC64_OP(and, &=) -ATOMIC64_OP(or, |=) -ATOMIC64_OP(xor, ^=) #undef ATOMIC64_OPS +#define ATOMIC64_OPS(op, c_op) \ + ATOMIC64_OP(op, c_op) \ + ATOMIC64_FETCH_OP(op, c_op) + +ATOMIC64_OPS(and, &=) +ATOMIC64_OPS(or, |=) +ATOMIC64_OPS(xor, ^=) + +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/parisc/include/asm/hash.h b/arch/parisc/include/asm/hash.h new file mode 100644 index 0000000..dbe9331 --- /dev/null +++ b/arch/parisc/include/asm/hash.h @@ -0,0 +1,146 @@ +#ifndef _ASM_HASH_H +#define _ASM_HASH_H + +/* + * HP-PA only implements integer multiply in the FPU. However, for + * integer multiplies by constant, it has a number of shift-and-add + * (but no shift-and-subtract, sigh!) instructions that a compiler + * can synthesize a code sequence with. + * + * Unfortunately, GCC isn't very efficient at using them. For example + * it uses three instructions for "x *= 21" when only two are needed. + * But we can find a sequence manually. + */ + +#define HAVE_ARCH__HASH_32 1 + +/* + * This is a multiply by GOLDEN_RATIO_32 = 0x61C88647 optimized for the + * PA7100 pairing rules. This is an in-order 2-way superscalar processor. + * Only one instruction in a pair may be a shift (by more than 3 bits), + * but other than that, simple ALU ops (including shift-and-add by up + * to 3 bits) may be paired arbitrarily. + * + * PA8xxx processors also dual-issue ALU instructions, although with + * fewer constraints, so this schedule is good for them, too. + * + * This 6-step sequence was found by Yevgen Voronenko's implementation + * of the Hcub algorithm at http://spiral.ece.cmu.edu/mcm/gen.html. + */ +static inline u32 __attribute_const__ __hash_32(u32 x) +{ + u32 a, b, c; + + /* + * Phase 1: Compute a = (x << 19) + x, + * b = (x << 9) + a, c = (x << 23) + b. + */ + a = x << 19; /* Two shifts can't be paired */ + b = x << 9; a += x; + c = x << 23; b += a; + c += b; + /* Phase 2: Return (b<<11) + (c<<6) + (a<<3) - c */ + b <<= 11; + a += c << 3; b -= c; + return (a << 3) + b; +} + +#if BITS_PER_LONG == 64 + +#define HAVE_ARCH_HASH_64 1 + +/* + * Finding a good shift-and-add chain for GOLDEN_RATIO_64 is tricky, + * because available software for the purpose chokes on constants this + * large. (It's mostly designed for compiling FIR filter coefficients + * into FPGAs.) + * + * However, Jason Thong pointed out a work-around. The Hcub software + * (http://spiral.ece.cmu.edu/mcm/gen.html) is designed for *multiple* + * constant multiplication, and is good at finding shift-and-add chains + * which share common terms. + * + * Looking at 0x0x61C8864680B583EB in binary: + * 0110000111001000100001100100011010000000101101011000001111101011 + * \______________/ \__________/ \_______/ \________/ + * \____________________________/ \____________________/ + * you can see the non-zero bits are divided into several well-separated + * blocks. Hcub can find algorithms for those terms separately, which + * can then be shifted and added together. + * + * Dividing the input into 2, 3 or 4 blocks, Hcub can find solutions + * with 10, 9 or 8 adds, respectively, making a total of 11 for the + * whole number. + * + * Using just two large blocks, 0xC3910C8D << 31 in the high bits, + * and 0xB583EB in the low bits, produces as good an algorithm as any, + * and with one more small shift than alternatives. + * + * The high bits are a larger number and more work to compute, as well + * as needing one extra cycle to shift left 31 bits before the final + * addition, so they are the critical path for scheduling. The low bits + * can fit into the scheduling slots left over. + */ + + +/* + * This _ASSIGN(dst, src) macro performs "dst = src", but prevents GCC + * from inferring anything about the value assigned to "dest". + * + * This prevents it from mis-optimizing certain sequences. + * In particular, gcc is annoyingly eager to combine consecutive shifts. + * Given "x <<= 19; y += x; z += x << 1;", GCC will turn this into + * "y += x << 19; z += x << 20;" even though the latter sequence needs + * an additional instruction and temporary register. + * + * Because no actual assembly code is generated, this construct is + * usefully portable across all GCC platforms, and so can be test-compiled + * on non-PA systems. + * + * In two places, additional unused input dependencies are added. This + * forces GCC's scheduling so it does not rearrange instructions too much. + * Because the PA-8xxx is out of order, I'm not sure how much this matters, + * but why make it more difficult for the processor than necessary? + */ +#define _ASSIGN(dst, src, ...) asm("" : "=r" (dst) : "0" (src), ##__VA_ARGS__) + +/* + * Multiply by GOLDEN_RATIO_64 = 0x0x61C8864680B583EB using a heavily + * optimized shift-and-add sequence. + * + * Without the final shift, the multiply proper is 19 instructions, + * 10 cycles and uses only 4 temporaries. Whew! + * + * You are not expected to understand this. + */ +static __always_inline u32 __attribute_const__ +hash_64(u64 a, unsigned int bits) +{ + u64 b, c, d; + + /* + * Encourage GCC to move a dynamic shift to %sar early, + * thereby freeing up an additional temporary register. + */ + if (!__builtin_constant_p(bits)) + asm("" : "=q" (bits) : "0" (64 - bits)); + else + bits = 64 - bits; + + _ASSIGN(b, a*5); c = a << 13; + b = (b << 2) + a; _ASSIGN(d, a << 17); + a = b + (a << 1); c += d; + d = a << 10; _ASSIGN(a, a << 19); + d = a - d; _ASSIGN(a, a << 4, "X" (d)); + c += b; a += b; + d -= c; c += a << 1; + a += c << 3; _ASSIGN(b, b << (7+31), "X" (c), "X" (d)); + a <<= 31; b += d; + a += b; + return a >> bits; +} +#undef _ASSIGN /* We're a widely-used header file, so don't litter! */ + +#endif /* BITS_PER_LONG == 64 */ + +#endif /* _ASM_HASH_H */ diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h index f2fd327..f08dda3 100644 --- a/arch/parisc/include/asm/pgalloc.h +++ b/arch/parisc/include/asm/pgalloc.h @@ -63,8 +63,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmd) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) { - pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, - PMD_ORDER); + pmd_t *pmd = (pmd_t *)__get_free_pages(GFP_KERNEL, PMD_ORDER); if (pmd) memset(pmd, 0, PAGE_SIZE<<PMD_ORDER); return pmd; @@ -124,7 +123,7 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + struct page *page = alloc_page(GFP_KERNEL|__GFP_ZERO); if (!page) return NULL; if (!pgtable_page_ctor(page)) { @@ -137,7 +136,7 @@ pte_alloc_one(struct mm_struct *mm, unsigned long address) static inline pte_t * pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); + pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); return pte; } diff --git a/arch/parisc/include/asm/spinlock.h b/arch/parisc/include/asm/spinlock.h index 64f2992..e32936c 100644 --- a/arch/parisc/include/asm/spinlock.h +++ b/arch/parisc/include/asm/spinlock.h @@ -13,8 +13,13 @@ static inline int arch_spin_is_locked(arch_spinlock_t *x) } #define arch_spin_lock(lock) arch_spin_lock_flags(lock, 0) -#define arch_spin_unlock_wait(x) \ - do { cpu_relax(); } while (arch_spin_is_locked(x)) + +static inline void arch_spin_unlock_wait(arch_spinlock_t *x) +{ + volatile unsigned int *a = __ldcw_align(x); + + smp_cond_load_acquire(a, VAL); +} static inline void arch_spin_lock_flags(arch_spinlock_t *x, unsigned long flags) diff --git a/arch/parisc/include/asm/traps.h b/arch/parisc/include/asm/traps.h index 4736020..5e953ab 100644 --- a/arch/parisc/include/asm/traps.h +++ b/arch/parisc/include/asm/traps.h @@ -8,6 +8,8 @@ struct pt_regs; void parisc_terminate(char *msg, struct pt_regs *regs, int code, unsigned long offset) __noreturn __cold; +void die_if_kernel(char *str, struct pt_regs *regs, long err); + /* mm/fault.c */ void do_page_fault(struct pt_regs *regs, unsigned long code, unsigned long address); |