diff options
Diffstat (limited to 'vec_mic_double8.h')
| -rw-r--r-- | vec_mic_double8.h | 1236 |
1 files changed, 562 insertions, 674 deletions
diff --git a/vec_mic_double8.h b/vec_mic_double8.h index 68dd5aa..ef22088 100644 --- a/vec_mic_double8.h +++ b/vec_mic_double8.h @@ -12,697 +12,585 @@ // MIC intrinsics #include <immintrin.h> - - namespace vecmathlib { - + #define VECMATHLIB_HAVE_VEC_DOUBLE_8 - template<> struct boolvec<double,8>; - template<> struct intvec<double,8>; - template<> struct realvec<double,8>; - - - - template<> - struct boolvec<double,8>: floatprops<double> - { - static const int size = 8; - typedef bool scalar_t; - typedef __mask8 bvector_t; - static const int alignment = sizeof(bvector_t); - - // static_assert(size * sizeof(real_t) == sizeof(bvector_t), - // "vector size is wrong"); - - typedef boolvec boolvec_t; - typedef intvec<real_t, size> intvec_t; - typedef realvec<real_t, size> realvec_t; - - // Short names for type casts - typedef real_t R; - typedef int_t I; - typedef uint_t U; - typedef realvec_t RV; - typedef intvec_t IV; - typedef boolvec_t BV; - typedef floatprops<real_t> FP; - typedef mathfuncs<realvec_t> MF; - - - - bvector_t v; - - boolvec() {} - // Can't have a non-trivial copy constructor; if so, objects won't - // be passed in registers - // boolvec(const boolvec& x): v(x.v) {} - // boolvec& operator=(const boolvec& x) { return v=x.v, *this; } - boolvec(bvector_t x): v(x) {} - boolvec(bool a): v(- bvector_t(a)) {} - boolvec(const bool* as): - v((bvector_t(as[0]) << 0) | - (bvector_t(as[1]) << 1) | - (bvector_t(as[2]) << 2) | - (bvector_t(as[3]) << 3) | - (bvector_t(as[4]) << 4) | - (bvector_t(as[5]) << 5) | - (bvector_t(as[6]) << 6) | - (bvector_t(as[7]) << 7)) - {} - - operator bvector_t() const { return v; } - bool operator[](int n) const - { - return (v >> n) & 1; - } - boolvec& set_elt(int n, bool a) - { - v &= ~ (bvector_t(1) << n); - v |= bvector_t(a) << n; - return *this; - } - - - - intvec_t as_int() const; // defined after intvec - intvec_t convert_int() const; // defined after intvec - - - - boolvec operator!() const { return _mm512_knot(v); } - - boolvec operator&&(boolvec x) const { return _mm512_kand(v, x.v); } - boolvec operator||(boolvec x) const { return _mm512_kor(v, x.v); } - boolvec operator==(boolvec x) const { return _mm512_kxnor(v, x.v); } - boolvec operator!=(boolvec x) const { return _mm512_kxor(v, x.v); } - - bool all() const { return _mm512_kortestc(v, v); } - bool any() const { return ! bool(_mm512_kortestz(v, v)); } - - - - // ifthen(condition, then-value, else-value) - boolvec_t ifthen(boolvec_t x, boolvec_t y) const; - intvec_t ifthen(intvec_t x, intvec_t y) const; // defined after intvec - realvec_t ifthen(realvec_t x, realvec_t y) const; // defined after realvec - }; - - - - template<> - struct intvec<double,8>: floatprops<double> - { - static const int size = 8; - typedef int_t scalar_t; - typedef __m512i ivector_t; - static const int alignment = sizeof(ivector_t); - - static_assert(size * sizeof(real_t) == sizeof(ivector_t), - "vector size is wrong"); - - typedef boolvec<real_t, size> boolvec_t; - typedef intvec intvec_t; - typedef realvec<real_t, size> realvec_t; - - // Short names for type casts - typedef real_t R; - typedef int_t I; - typedef uint_t U; - typedef realvec_t RV; - typedef intvec_t IV; - typedef boolvec_t BV; - typedef floatprops<real_t> FP; - typedef mathfuncs<realvec_t> MF; - - - - ivector_t v; - - intvec() {} - // Can't have a non-trivial copy constructor; if so, objects won't - // be passed in registers - // intvec(const intvec& x): v(x.v) {} - // intvec& operator=(const intvec& x) { return v=x.v, *this; } - intvec(ivector_t x): v(x) {} - intvec(int_t a): v(_mm512_set1_epi64(a)) {} - intvec(const int_t* as) - { - v = _mm512_undefined_epi32(); - // v = _mm512_loadunpacklo_epi32(v, as); - // v = _mm512_loadunpackhi_epi32(v, as+8); - for (int n=0; n<size; ++n) set_elt(n, as[n]); - } - static intvec iota() - { - intvec r; - for (int n=0; n<size; ++n) r.set_elt(n, n); - return r; - } - - operator ivector_t() const { return v; } - int_t operator[](int n) const - { - return vecmathlib::get_elt<IV,ivector_t,int_t>(v, n); - } - intvec_t& set_elt(int n, int_t a) - { - return vecmathlib::set_elt<IV,ivector_t,int_t>(v, n, a), *this; - } - - - - private: - static __mmask8 mask16tomask8(__mmask16 m16) - { - // combine 01 - m16 = ((m16 >> 1) | m16) & 0b0011001100110011; - // combine 0123 - m16 = ((m16 >> 2) | m16) & 0b0000111100001111; - // combine 01234567 - m16 = ((m16 >> 4) | m16) & 0b0000000011111111; - return m16; - } - public: - boolvec_t as_bool() const { return convert_bool(); } - boolvec_t convert_bool() const - { - // Result: convert_bool(0)=false, convert_bool(else)=true - __mmask16 r16 = _mm512_test_epi32_mask(v, v); - return mask16tomask8(r16); - } - realvec_t as_float() const; // defined after realvec - realvec_t convert_float() const; // defined after realvec - - - - // Note: not all arithmetic operations are supported! - - intvec operator+() const { return *this; } - intvec operator-() const { return IV(I(0)) - *this; } - intvec operator+(intvec x) const { return _mm512_add_epi64(v, x.v); } - intvec operator-(intvec x) const { return _mm512_sub_epi64(v, x.v); } - - intvec& operator+=(const intvec& x) { return *this=*this+x; } - intvec& operator-=(const intvec& x) { return *this=*this-x; } - - - - intvec operator~() const { return IV(~U(0)) ^ *this; } - intvec operator&(intvec x) const { return _mm512_and_epi64(v, x.v); } - intvec operator|(intvec x) const { return _mm512_or_epi64(v, x.v); } - intvec operator^(intvec x) const { return _mm512_xor_epi64(v, x.v); } - - intvec& operator&=(const intvec& x) { return *this=*this&x; } - intvec& operator|=(const intvec& x) { return *this=*this|x; } - intvec& operator^=(const intvec& x) { return *this=*this^x; } - - intvec_t bitifthen(intvec_t x, intvec_t y) const; - - - - intvec lsr(int_t n) const - { - if (n < 32) { - __m512i vlo = _mm512_srli_epi32(v, n); - __m512i vhi = _mm512_slli_epi32(v, 32-n); - vhi = _mm512_swizzle_epi32(vhi, _MM_SWIZ_REG_CDAB); - return _mm512_mask_or_epi32(vlo, 0xb0101010101010101, vhi, vlo); - } else { - __m512i vlo = _mm512_srli_epi32(v, n-32); - __m512i vhi = _mm512_setzero_epi32(); - return _mm512_mask_swizzle_epi32(vhi, 0xb0101010101010101, vlo); - } - } - intvec_t rotate(int_t n) const; - intvec operator>>(int_t n) const - { - if (n < 32) { - __mm512i vlo = _mm512_srai_epi32(v, n); - __mm512i vlo0 = _mm512_srli_epi32(v, n); - __mm512i vhi = _mm512_slli_epi32(v, 32-n); - vhi = _mm512_swizzle_epi32(vhi, _MM_SWIZ_REG_CDAB); - return _mm512_mask_or_epi32(vlo, 0xb0101010101010101, vhi, vlo0); - } else { - __m512i vlo = _mm512_srai_epi32(v, n-32); - __m512i vhi = _mm512_srai_epi32(v, 31); - return _mm512_mask_swizzle_epi32(vhi, 0xb0101010101010101, vlo); - } - } - intvec operator<<(int_t n) const - { - if (n < 32) { - __m512i vlo = _mm512_srli_epi32(v, n); - __m512i vhi = _mm512_slli_epi32(v, 32-n); - vlo = _mm512_swizzle_epi32(vlo, _MM_SWIZ_REG_CDAB); - return _mm512_mask_or_epi32(vhi, 0xb1010101010101010, vhi, vlo); - } else { - __m512i vlo = _mm512_setzero_epi32(); - __m512i vhi = _mm512_slli_epi32(v, n-32); - return _mm512_mask_swizzle_epi32(vhi, 0xb1010101010101010, vlo); - } - } - intvec& operator>>=(int_t n) { return *this=*this>>n; } - intvec& operator<<=(int_t n) { return *this=*this<<n; } - - intvec lsr(intvec n) const - { - // TODO: improve this - intvec r; - for (int i=0; i<size; ++i) { - r.set_elt(i, U((*this)[i]) >> U(n[i])); - } - return r; - } - intvec_t rotate(intvec_t n) const; - intvec operator>>(intvec n) const - { - // TODO: improve this - intvec r; - for (int i=0; i<size; ++i) { - r.set_elt(i, (*this)[i] >> n[i]); - } - return r; - } - intvec operator<<(intvec n) const - { - // TODO: improve this - intvec r; - for (int i=0; i<size; ++i) { - r.set_elt(i, (*this)[i] << n[i]); - } - return r; - } - intvec& operator>>=(intvec n) { return *this=*this>>n; } - intvec& operator<<=(intvec n) { return *this=*this<<n; } - - intvec_t clz() const - { - // Return 8*sizeof(TYPE) when the input is 0 - intvec_t r; - for (int i=0; i<size; ++i) { - // __lzcnt64 - r.set_elt(i, __builtin_clzll((*this)[i])); - } - return r; - } - intvec_t popcount() const - { - intvec_t r; - for (int i=0; i<size; ++i) { - // _mm_popcnt_u64 - r.set_elt(i, __builtin_popcountll((*this)[i])); - } - return r; - } - - - - boolvec_t operator==(const intvec& x) const - { - return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_EQ)); - } - boolvec_t operator!=(const intvec& x) const - { - return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_NE)); - } - boolvec_t operator<(const intvec& x) const - { - return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_LT)); - } - boolvec_t operator<=(const intvec& x) const - { - return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_LE)); - } - boolvec_t operator>(const intvec& x) const - { - return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_GT)); - } - boolvec_t operator>=(const intvec& x) const - { - return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_GE)); - } - - intvec_t abs() const; - boolvec_t isignbit() const; - intvec_t max(intvec_t x) const; - intvec_t min(intvec_t x) const; - }; - - - - template<> - struct realvec<double,8>: floatprops<double> - { - static const int size = 8; - typedef real_t scalar_t; - typedef __m512d vector_t; - static const int alignment = sizeof(vector_t); - - static const char* name() { return "<MIC:8*double>"; } - void barrier() { __asm__("": "+x"(v)); } - - static_assert(size * sizeof(real_t) == sizeof(vector_t), - "vector size is wrong"); - - typedef boolvec<real_t, size> boolvec_t; - typedef intvec<real_t, size> intvec_t; - typedef realvec realvec_t; - - // Short names for type casts - typedef real_t R; - typedef int_t I; - typedef uint_t U; - typedef realvec_t RV; - typedef intvec_t IV; - typedef boolvec_t BV; - typedef floatprops<real_t> FP; - typedef mathfuncs<realvec_t> MF; - - - - vector_t v; - - realvec() {} - // Can't have a non-trivial copy constructor; if so, objects won't - // be passed in registers - // realvec(const realvec& x): v(x.v) {} - // realvec& operator=(const realvec& x) { return v=x.v, *this; } - realvec(vector_t x): v(x) {} - realvec(real_t a): v(_mm512_set1_pd(a)) {} - realvec(const real_t* as) - { - v = _mm512_undefined_pd(); - // v = _mm512_loadunpacklo_pd(v, as); - // v = _mm512_loadunpackhi_pd(v, as+8); - for (int n=0; n<size; ++n) set_elt(n, as[n]); - } - - operator vector_t() const { return v; } - real_t operator[](int n) const - { - return vecmathlib::get_elt<RV,vector_t,real_t>(v, n); - } - realvec_t& set_elt(int n, real_t a) - { - return vecmathlib::set_elt<RV,vector_t,real_t>(v, n, a), *this; - } - - - - typedef vecmathlib::mask_t<realvec_t> mask_t; - - static realvec_t loada(const real_t* p) - { - VML_ASSERT(intptr_t(p) % alignment == 0); - return _mm512_load_pd(p); - } - static realvec_t loadu(const real_t* p) - { - realvec_t r(_mm512_undefined_pd()); - r.v = _mm512_loadunpacklo_pd(r.v, p); - r.v = _mm512_loadunpackhi_pd(r.v, p+8); - return r.v; - } - static realvec_t loadu(const real_t* p, std::ptrdiff_t ioff) - { - VML_ASSERT(intptr_t(p) % alignment == 0); - if (ioff % realvec::size == 0) return loada(p+ioff); - return loadu(p+ioff); - } - realvec_t loada(const real_t* p, const mask_t& m) const - { - VML_ASSERT(intptr_t(p) % alignment == 0); - return _mm512_mask_load_pd(v, m.m.v, p); +template <> struct boolvec<double, 8>; +template <> struct intvec<double, 8>; +template <> struct realvec<double, 8>; + +template <> struct boolvec<double, 8> : floatprops<double> { + static const int size = 8; + typedef bool scalar_t; + typedef __mask8 bvector_t; + static const int alignment = sizeof(bvector_t); + + // static_assert(size * sizeof(real_t) == sizeof(bvector_t), + // "vector size is wrong"); + + typedef boolvec boolvec_t; + typedef intvec<real_t, size> intvec_t; + typedef realvec<real_t, size> realvec_t; + + // Short names for type casts + typedef real_t R; + typedef int_t I; + typedef uint_t U; + typedef realvec_t RV; + typedef intvec_t IV; + typedef boolvec_t BV; + typedef floatprops<real_t> FP; + typedef mathfuncs<realvec_t> MF; + + bvector_t v; + + boolvec() {} + // Can't have a non-trivial copy constructor; if so, objects won't + // be passed in registers + // boolvec(const boolvec& x): v(x.v) {} + // boolvec& operator=(const boolvec& x) { return v=x.v, *this; } + boolvec(bvector_t x) : v(x) {} + boolvec(bool a) : v(-bvector_t(a)) {} + boolvec(const bool *as) + : v((bvector_t(as[0]) << 0) | (bvector_t(as[1]) << 1) | + (bvector_t(as[2]) << 2) | (bvector_t(as[3]) << 3) | + (bvector_t(as[4]) << 4) | (bvector_t(as[5]) << 5) | + (bvector_t(as[6]) << 6) | (bvector_t(as[7]) << 7)) {} + + operator bvector_t() const { return v; } + bool operator[](int n) const { return (v >> n) & 1; } + boolvec &set_elt(int n, bool a) { + v &= ~(bvector_t(1) << n); + v |= bvector_t(a) << n; + return *this; + } + + intvec_t as_int() const; // defined after intvec + intvec_t convert_int() const; // defined after intvec + + boolvec operator!() const { return _mm512_knot(v); } + + boolvec operator&&(boolvec x) const { return _mm512_kand(v, x.v); } + boolvec operator||(boolvec x) const { return _mm512_kor(v, x.v); } + boolvec operator==(boolvec x) const { return _mm512_kxnor(v, x.v); } + boolvec operator!=(boolvec x) const { return _mm512_kxor(v, x.v); } + + bool all() const { return _mm512_kortestc(v, v); } + bool any() const { return !bool(_mm512_kortestz(v, v)); } + + // ifthen(condition, then-value, else-value) + boolvec_t ifthen(boolvec_t x, boolvec_t y) const; + intvec_t ifthen(intvec_t x, intvec_t y) const; // defined after intvec + realvec_t ifthen(realvec_t x, realvec_t y) const; // defined after realvec +}; + +template <> struct intvec<double, 8> : floatprops<double> { + static const int size = 8; + typedef int_t scalar_t; + typedef __m512i ivector_t; + static const int alignment = sizeof(ivector_t); + + static_assert(size * sizeof(real_t) == sizeof(ivector_t), + "vector size is wrong"); + + typedef boolvec<real_t, size> boolvec_t; + typedef intvec intvec_t; + typedef realvec<real_t, size> realvec_t; + + // Short names for type casts + typedef real_t R; + typedef int_t I; + typedef uint_t U; + typedef realvec_t RV; + typedef intvec_t IV; + typedef boolvec_t BV; + typedef floatprops<real_t> FP; + typedef mathfuncs<realvec_t> MF; + + ivector_t v; + + intvec() {} + // Can't have a non-trivial copy constructor; if so, objects won't + // be passed in registers + // intvec(const intvec& x): v(x.v) {} + // intvec& operator=(const intvec& x) { return v=x.v, *this; } + intvec(ivector_t x) : v(x) {} + intvec(int_t a) : v(_mm512_set1_epi64(a)) {} + intvec(const int_t *as) { + v = _mm512_undefined_epi32(); + // v = _mm512_loadunpacklo_epi32(v, as); + // v = _mm512_loadunpackhi_epi32(v, as+8); + for (int n = 0; n < size; ++n) + set_elt(n, as[n]); + } + static intvec iota() { + intvec r; + for (int n = 0; n < size; ++n) + r.set_elt(n, n); + return r; + } + + operator ivector_t() const { return v; } + int_t operator[](int n) const { + return vecmathlib::get_elt<IV, ivector_t, int_t>(v, n); + } + intvec_t &set_elt(int n, int_t a) { + return vecmathlib::set_elt<IV, ivector_t, int_t>(v, n, a), *this; + } + +private: + static __mmask8 mask16tomask8(__mmask16 m16) { + // combine 01 + m16 = ((m16 >> 1) | m16) & 0b0011001100110011; + // combine 0123 + m16 = ((m16 >> 2) | m16) & 0b0000111100001111; + // combine 01234567 + m16 = ((m16 >> 4) | m16) & 0b0000000011111111; + return m16; + } + +public: + boolvec_t as_bool() const { return convert_bool(); } + boolvec_t convert_bool() const { + // Result: convert_bool(0)=false, convert_bool(else)=true + __mmask16 r16 = _mm512_test_epi32_mask(v, v); + return mask16tomask8(r16); + } + realvec_t as_float() const; // defined after realvec + realvec_t convert_float() const; // defined after realvec + + // Note: not all arithmetic operations are supported! + + intvec operator+() const { return *this; } + intvec operator-() const { return IV(I(0)) - *this; } + intvec operator+(intvec x) const { return _mm512_add_epi64(v, x.v); } + intvec operator-(intvec x) const { return _mm512_sub_epi64(v, x.v); } + + intvec &operator+=(const intvec &x) { return *this = *this + x; } + intvec &operator-=(const intvec &x) { return *this = *this - x; } + + intvec operator~() const { return IV(~U(0)) ^ *this; } + intvec operator&(intvec x) const { return _mm512_and_epi64(v, x.v); } + intvec operator|(intvec x) const { return _mm512_or_epi64(v, x.v); } + intvec operator^(intvec x) const { return _mm512_xor_epi64(v, x.v); } + + intvec &operator&=(const intvec &x) { return *this = *this & x; } + intvec &operator|=(const intvec &x) { return *this = *this | x; } + intvec &operator^=(const intvec &x) { return *this = *this ^ x; } + + intvec_t bitifthen(intvec_t x, intvec_t y) const; + + intvec lsr(int_t n) const { + if (n < 32) { + __m512i vlo = _mm512_srli_epi32(v, n); + __m512i vhi = _mm512_slli_epi32(v, 32 - n); + vhi = _mm512_swizzle_epi32(vhi, _MM_SWIZ_REG_CDAB); + return _mm512_mask_or_epi32(vlo, 0xb0101010101010101, vhi, vlo); + } else { + __m512i vlo = _mm512_srli_epi32(v, n - 32); + __m512i vhi = _mm512_setzero_epi32(); + return _mm512_mask_swizzle_epi32(vhi, 0xb0101010101010101, vlo); } - realvec_t loadu(const real_t* p, const mask_t& m) const - { - if (__builtin_expect(m.all_m, true)) { - return loadu(p); - } else { - return m.m.ifthen(loadu(p), *this); - } + } + intvec_t rotate(int_t n) const; + intvec operator>>(int_t n) const { + if (n < 32) { + __mm512i vlo = _mm512_srai_epi32(v, n); + __mm512i vlo0 = _mm512_srli_epi32(v, n); + __mm512i vhi = _mm512_slli_epi32(v, 32 - n); + vhi = _mm512_swizzle_epi32(vhi, _MM_SWIZ_REG_CDAB); + return _mm512_mask_or_epi32(vlo, 0xb0101010101010101, vhi, vlo0); + } else { + __m512i vlo = _mm512_srai_epi32(v, n - 32); + __m512i vhi = _mm512_srai_epi32(v, 31); + return _mm512_mask_swizzle_epi32(vhi, 0xb0101010101010101, vlo); } - realvec_t loadu(const real_t* p, std::ptrdiff_t ioff, const mask_t& m) const - { - VML_ASSERT(intptr_t(p) % alignment == 0); - if (ioff % realvec::size == 0) return loada(p+ioff, m); - return loadu(p+ioff, m); + } + intvec operator<<(int_t n) const { + if (n < 32) { + __m512i vlo = _mm512_srli_epi32(v, n); + __m512i vhi = _mm512_slli_epi32(v, 32 - n); + vlo = _mm512_swizzle_epi32(vlo, _MM_SWIZ_REG_CDAB); + return _mm512_mask_or_epi32(vhi, 0xb1010101010101010, vhi, vlo); + } else { + __m512i vlo = _mm512_setzero_epi32(); + __m512i vhi = _mm512_slli_epi32(v, n - 32); + return _mm512_mask_swizzle_epi32(vhi, 0xb1010101010101010, vlo); } - - void storea(real_t* p) const - { - VML_ASSERT(intptr_t(p) % alignment == 0); - _mm512_store_pd(p, v); + } + intvec &operator>>=(int_t n) { return *this = *this >> n; } + intvec &operator<<=(int_t n) { return *this = *this << n; } + + intvec lsr(intvec n) const { + // TODO: improve this + intvec r; + for (int i = 0; i < size; ++i) { + r.set_elt(i, U((*this)[i]) >> U(n[i])); } - void storeu(real_t* p) const - { - _mm512_packstorelo_pd(p, v); - _mm512_packstorehi_pd(p+8, v); + return r; + } + intvec_t rotate(intvec_t n) const; + intvec operator>>(intvec n) const { + // TODO: improve this + intvec r; + for (int i = 0; i < size; ++i) { + r.set_elt(i, (*this)[i] >> n[i]); } - void storeu(real_t* p, std::ptrdiff_t ioff) const - { - VML_ASSERT(intptr_t(p) % alignment == 0); - if (ioff % realvec::size == 0) return storea(p+ioff); - storeu(p+ioff); + return r; + } + intvec operator<<(intvec n) const { + // TODO: improve this + intvec r; + for (int i = 0; i < size; ++i) { + r.set_elt(i, (*this)[i] << n[i]); } - void storea(real_t* p, const mask_t& m) const - { - VML_ASSERT(intptr_t(p) % alignment == 0); - _mm512_mask_store_pd(p, m.m.v, v); + return r; + } + intvec &operator>>=(intvec n) { return *this = *this >> n; } + intvec &operator<<=(intvec n) { return *this = *this << n; } + + intvec_t clz() const { + // Return 8*sizeof(TYPE) when the input is 0 + intvec_t r; + for (int i = 0; i < size; ++i) { + // __lzcnt64 + r.set_elt(i, __builtin_clzll((*this)[i])); } - void storeu(real_t* p, const mask_t& m) const - { - if (__builtin_expect(m.all_m, true)) { - storeu(p); - } else { - for (int n=0; n<size; ++n) { - if (m.m[n]) p[n] = (*this)[n]; - } - } + return r; + } + intvec_t popcount() const { + intvec_t r; + for (int i = 0; i < size; ++i) { + // _mm_popcnt_u64 + r.set_elt(i, __builtin_popcountll((*this)[i])); } - void storeu(real_t* p, std::ptrdiff_t ioff, const mask_t& m) const - { - VML_ASSERT(intptr_t(p) % alignment == 0); - if (ioff % realvec::size == 0) return storea(p+ioff, m); - storeu(p+ioff, m); + return r; + } + + boolvec_t operator==(const intvec &x) const { + return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_EQ)); + } + boolvec_t operator!=(const intvec &x) const { + return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_NE)); + } + boolvec_t operator<(const intvec &x) const { + return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_LT)); + } + boolvec_t operator<=(const intvec &x) const { + return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_LE)); + } + boolvec_t operator>(const intvec &x) const { + return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_GT)); + } + boolvec_t operator>=(const intvec &x) const { + return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_GE)); + } + + intvec_t abs() const; + boolvec_t isignbit() const; + intvec_t max(intvec_t x) const; + intvec_t min(intvec_t x) const; +}; + +template <> struct realvec<double, 8> : floatprops<double> { + static const int size = 8; + typedef real_t scalar_t; + typedef __m512d vector_t; + static const int alignment = sizeof(vector_t); + + static const char *name() { return "<MIC:8*double>"; } + void barrier() { __asm__("" : "+x"(v)); } + + static_assert(size * sizeof(real_t) == sizeof(vector_t), + "vector size is wrong"); + + typedef boolvec<real_t, size> boolvec_t; + typedef intvec<real_t, size> intvec_t; + typedef realvec realvec_t; + + // Short names for type casts + typedef real_t R; + typedef int_t I; + typedef uint_t U; + typedef realvec_t RV; + typedef intvec_t IV; + typedef boolvec_t BV; + typedef floatprops<real_t> FP; + typedef mathfuncs<realvec_t> MF; + + vector_t v; + + realvec() {} + // Can't have a non-trivial copy constructor; if so, objects won't + // be passed in registers + // realvec(const realvec& x): v(x.v) {} + // realvec& operator=(const realvec& x) { return v=x.v, *this; } + realvec(vector_t x) : v(x) {} + realvec(real_t a) : v(_mm512_set1_pd(a)) {} + realvec(const real_t *as) { + v = _mm512_undefined_pd(); + // v = _mm512_loadunpacklo_pd(v, as); + // v = _mm512_loadunpackhi_pd(v, as+8); + for (int n = 0; n < size; ++n) + set_elt(n, as[n]); + } + + operator vector_t() const { return v; } + real_t operator[](int n) const { + return vecmathlib::get_elt<RV, vector_t, real_t>(v, n); + } + realvec_t &set_elt(int n, real_t a) { + return vecmathlib::set_elt<RV, vector_t, real_t>(v, n, a), *this; + } + + typedef vecmathlib::mask_t<realvec_t> mask_t; + + static realvec_t loada(const real_t *p) { + VML_ASSERT(intptr_t(p) % alignment == 0); + return _mm512_load_pd(p); + } + static realvec_t loadu(const real_t *p) { + realvec_t r(_mm512_undefined_pd()); + r.v = _mm512_loadunpacklo_pd(r.v, p); + r.v = _mm512_loadunpackhi_pd(r.v, p + 8); + return r.v; + } + static realvec_t loadu(const real_t *p, std::ptrdiff_t ioff) { + VML_ASSERT(intptr_t(p) % alignment == 0); + if (ioff % realvec::size == 0) + return loada(p + ioff); + return loadu(p + ioff); + } + realvec_t loada(const real_t *p, const mask_t &m) const { + VML_ASSERT(intptr_t(p) % alignment == 0); + return _mm512_mask_load_pd(v, m.m.v, p); + } + realvec_t loadu(const real_t *p, const mask_t &m) const { + if (__builtin_expect(m.all_m, true)) { + return loadu(p); + } else { + return m.m.ifthen(loadu(p), *this); } - - - - intvec_t as_int() const { return _mm512_castpd_si512(v); } - intvec_t convert_int() const - { - intvec_t r(_mm512_undefined_epi32()); - for (int n=0; n<size; ++n) { - r.set_elt(n, floatprops::convert_int((*this)[n])); + } + realvec_t loadu(const real_t *p, std::ptrdiff_t ioff, const mask_t &m) const { + VML_ASSERT(intptr_t(p) % alignment == 0); + if (ioff % realvec::size == 0) + return loada(p + ioff, m); + return loadu(p + ioff, m); + } + + void storea(real_t *p) const { + VML_ASSERT(intptr_t(p) % alignment == 0); + _mm512_store_pd(p, v); + } + void storeu(real_t *p) const { + _mm512_packstorelo_pd(p, v); + _mm512_packstorehi_pd(p + 8, v); + } + void storeu(real_t *p, std::ptrdiff_t ioff) const { + VML_ASSERT(intptr_t(p) % alignment == 0); + if (ioff % realvec::size == 0) + return storea(p + ioff); + storeu(p + ioff); + } + void storea(real_t *p, const mask_t &m) const { + VML_ASSERT(intptr_t(p) % alignment == 0); + _mm512_mask_store_pd(p, m.m.v, v); + } + void storeu(real_t *p, const mask_t &m) const { + if (__builtin_expect(m.all_m, true)) { + storeu(p); + } else { + for (int n = 0; n < size; ++n) { + if (m.m[n]) + p[n] = (*this)[n]; } - return r; - } - - - - realvec operator+() const { return *this; } - realvec operator-() const { return RV(0.0) - *this; } - - realvec operator+(realvec x) const { return _mm512_add_pd(v, x.v); } - realvec operator-(realvec x) const { return _mm512_sub_pd(v, x.v); } - realvec operator*(realvec x) const { return _mm512_mul_pd(v, x.v); } - realvec operator/(realvec x) const { return _mm512_div_pd(v, x.v); } - - realvec& operator+=(const realvec& x) { return *this=*this+x; } - realvec& operator-=(const realvec& x) { return *this=*this-x; } - realvec& operator*=(const realvec& x) { return *this=*this*x; } - realvec& operator/=(const realvec& x) { return *this=*this/x; } - - real_t maxval() const { returm _mm512_reduce_gmax_pd(v); } - real_t minval() const { returm _mm512_reduce_gmin_pd(v); } - real_t prod() const { returm _mm512_reduce_mul_pd(v); } - real_t sum() const { returm _mm512_reduce_add_pd(v); } - - - - boolvec_t operator==(const realvec& x) const - { - return _mm512_cmp_pd(v, x.v, _CMP_EQ_OQ); - } - boolvec_t operator!=(const realvec& x) const - { - return _mm512_cmp_pd(v, x.v, _CMP_NEQ_UQ); // Note: _UQ here - } - boolvec_t operator<(const realvec& x) const - { - return _mm512_cmp_pd(v, x.v, _CMP_LT_OQ); - } - boolvec_t operator<=(const realvec& x) const - { - return _mm512_cmp_pd(v, x.v, _CMP_LE_OQ); - } - boolvec_t operator>(const realvec& x) const - { - return _mm512_cmp_pd(v, x.v, _CMP_GT_OQ); } - boolvec_t operator>=(const realvec& x) const - { - return _mm512_cmp_pd(v, x.v, _CMP_GE_OQ); - } - - - - realvec acos() const { return MF::vml_acos(*this); } - realvec acosh() const { return MF::vml_acosh(*this); } - realvec asin() const { return MF::vml_asin(*this); } - realvec asinh() const { return MF::vml_asinh(*this); } - realvec atan() const { return MF::vml_atan(*this); } - realvec atan2(realvec y) const { return MF::vml_atan2(*this, y); } - realvec atanh() const { return MF::vml_atanh(*this); } - realvec cbrt() const { return MF::vml_cbrt(*this); } - realvec ceil() const { return _mm512_ceil_pd(v); } - realvec copysign(realvec y) const { return MF::vml_copysign(*this, y); } - realvec cos() const { return MF::vml_cos(*this); } - realvec cosh() const { return MF::vml_cosh(*this); } - realvec exp() const { return MF::vml_exp(*this); } - realvec exp10() const { return MF::vml_exp10(*this); } - realvec exp2() const { return MF::vml_exp2(*this); } - realvec expm1() const { return MF::vml_expm1(*this); } - realvec fabs() const { return MF::vml_fabs(*this); } - realvec fdim(realvec y) const { return MF::vml_fdim(*this, y); } - realvec floor() const { return _mm512_floor_pd(v); } - realvec fma(realvec y, realvec z) const - { - return _mm512_fmadd_pd(v, x.v, y.v); + } + void storeu(real_t *p, std::ptrdiff_t ioff, const mask_t &m) const { + VML_ASSERT(intptr_t(p) % alignment == 0); + if (ioff % realvec::size == 0) + return storea(p + ioff, m); + storeu(p + ioff, m); + } + + intvec_t as_int() const { return _mm512_castpd_si512(v); } + intvec_t convert_int() const { + intvec_t r(_mm512_undefined_epi32()); + for (int n = 0; n < size; ++n) { + r.set_elt(n, floatprops::convert_int((*this)[n])); } - realvec fmax(realvec y) const { return _mm512_gmax_pd(v, y.v); } - realvec fmin(realvec y) const { return _mm512_gmin_pd(v, y.v); } - realvec fmod(realvec y) const { return MF::vml_fmod(*this, y); } - realvec frexp(intvec_t* r) const { return MF::vml_frexp(*this, r); } - realvec hypot(realvec y) const { return MF::vml_hypot(*this, y); } - intvec_t ilogb() const { return MF::vml_ilogb(*this); } - boolvec_t isfinite() const { return MF::vml_isfinite(*this); } - boolvec_t isinf() const { return MF::vml_isinf(*this); } - boolvec_t isnan() const - { + return r; + } + + realvec operator+() const { return *this; } + realvec operator-() const { return RV(0.0) - *this; } + + realvec operator+(realvec x) const { return _mm512_add_pd(v, x.v); } + realvec operator-(realvec x) const { return _mm512_sub_pd(v, x.v); } + realvec operator*(realvec x) const { return _mm512_mul_pd(v, x.v); } + realvec operator/(realvec x) const { return _mm512_div_pd(v, x.v); } + + realvec &operator+=(const realvec &x) { return *this = *this + x; } + realvec &operator-=(const realvec &x) { return *this = *this - x; } + realvec &operator*=(const realvec &x) { return *this = *this * x; } + realvec &operator/=(const realvec &x) { return *this = *this / x; } + + real_t maxval() const { returm _mm512_reduce_gmax_pd(v); } + real_t minval() const { returm _mm512_reduce_gmin_pd(v); } + real_t prod() const { returm _mm512_reduce_mul_pd(v); } + real_t sum() const { returm _mm512_reduce_add_pd(v); } + + boolvec_t operator==(const realvec &x) const { + return _mm512_cmp_pd(v, x.v, _CMP_EQ_OQ); + } + boolvec_t operator!=(const realvec &x) const { + return _mm512_cmp_pd(v, x.v, _CMP_NEQ_UQ); // Note: _UQ here + } + boolvec_t operator<(const realvec &x) const { + return _mm512_cmp_pd(v, x.v, _CMP_LT_OQ); + } + boolvec_t operator<=(const realvec &x) const { + return _mm512_cmp_pd(v, x.v, _CMP_LE_OQ); + } + boolvec_t operator>(const realvec &x) const { + return _mm512_cmp_pd(v, x.v, _CMP_GT_OQ); + } + boolvec_t operator>=(const realvec &x) const { + return _mm512_cmp_pd(v, x.v, _CMP_GE_OQ); + } + + realvec acos() const { return MF::vml_acos(*this); } + realvec acosh() const { return MF::vml_acosh(*this); } + realvec asin() const { return MF::vml_asin(*this); } + realvec asinh() const { return MF::vml_asinh(*this); } + realvec atan() const { return MF::vml_atan(*this); } + realvec atan2(realvec y) const { return MF::vml_atan2(*this, y); } + realvec atanh() const { return MF::vml_atanh(*this); } + realvec cbrt() const { return MF::vml_cbrt(*this); } + realvec ceil() const { return _mm512_ceil_pd(v); } + realvec copysign(realvec y) const { return MF::vml_copysign(*this, y); } + realvec cos() const { return MF::vml_cos(*this); } + realvec cosh() const { return MF::vml_cosh(*this); } + realvec exp() const { return MF::vml_exp(*this); } + realvec exp10() const { return MF::vml_exp10(*this); } + realvec exp2() const { return MF::vml_exp2(*this); } + realvec expm1() const { return MF::vml_expm1(*this); } + realvec fabs() const { return MF::vml_fabs(*this); } + realvec fdim(realvec y) const { return MF::vml_fdim(*this, y); } + realvec floor() const { return _mm512_floor_pd(v); } + realvec fma(realvec y, realvec z) const { + return _mm512_fmadd_pd(v, x.v, y.v); + } + realvec fmax(realvec y) const { return _mm512_gmax_pd(v, y.v); } + realvec fmin(realvec y) const { return _mm512_gmin_pd(v, y.v); } + realvec fmod(realvec y) const { return MF::vml_fmod(*this, y); } + realvec frexp(intvec_t *r) const { return MF::vml_frexp(*this, r); } + realvec hypot(realvec y) const { return MF::vml_hypot(*this, y); } + intvec_t ilogb() const { return MF::vml_ilogb(*this); } + boolvec_t isfinite() const { return MF::vml_isfinite(*this); } + boolvec_t isinf() const { return MF::vml_isinf(*this); } + boolvec_t isnan() const { #ifdef VML_HAVE_NAN - return _mm512_cmp_pd(v, v, _CMP_UNORD_Q); + return _mm512_cmp_pd(v, v, _CMP_UNORD_Q); #else - return BV(false); + return BV(false); #endif - } - boolvec_t isnormal() const { return MF::vml_isnormal(*this); } - realvec ldexp(int_t n) const { return MF::vml_ldexp(*this, n); } - realvec ldexp(intvec_t n) const { return MF::vml_ldexp(*this, n); } - realvec log() const { return MF::vml_log(*this); } - realvec log10() const { return MF::vml_log10(*this); } - realvec log1p() const { return MF::vml_log1p(*this); } - realvec log2() const { return MF::vml_log2(*this); } - realvec_t mad(realvec_t y, realvec_t z) const - { - return _mm512_fmadd_pd(v, x.v, y.v); - } - realvec nextafter(realvec y) const { return MF::vml_nextafter(*this, y); } - realvec pow(realvec y) const { return MF::vml_pow(*this, y); } - realvec rcp() const { return _mm512_div_pd(_mm512_set1_pd(1.0), v); } - realvec remainder(realvec y) const { return MF::vml_remainder(*this, y); } - realvec rint() const - { - return _mm512_round_pd(v, _MM_FROUND_TO_NEAREST_INT); - } - realvec round() const { return MF::vml_round(*this); } - realvec rsqrt() const { return MF::vml_rsqrt(*this); } - boolvec_t signbit() const { return as_int().signbit(); } - realvec sin() const { return MF::vml_sin(*this); } - realvec sinh() const { return MF::vml_sinh(*this); } - realvec sqrt() const { return _mm512_sqrt_pd(v); } - realvec tan() const { return MF::vml_tan(*this); } - realvec tanh() const { return MF::vml_tanh(*this); } - realvec trunc() const { return _mm512_round_pd(v, _MM_FROUND_TO_ZERO); } - }; - - - - // boolvec definitions - - inline intvec<double,4> boolvec<double,4>::as_int() const - { - return _mm512_castpd_si512(v); - } - - inline intvec<double,4> boolvec<double,4>::convert_int() const - { - return ifthen(v, IV(I(1)), IV(I(0))); - } - - inline - boolvec<double,4> boolvec<double,4>::ifthen(boolvec_t x, boolvec_t y) const - { - return (v & x.v) | (~v & y.v); - } - - inline - intvec<double,4> boolvec<double,4>::ifthen(intvec_t x, intvec_t y) const - { - return _mm512_blend_epi64(v, y.v, x.v) - } - - inline - realvec<double,4> boolvec<double,4>::ifthen(realvec_t x, realvec_t y) const - { - return _mm512_blend_pd(v, y.v, x.v) - } - - - - // intvec definitions - - inline realvec<double,4> intvec<double,4>::as_float() const - { - return _mm512_castsi512_pd(v); - } - - inline realvec<double,4> intvec<double,4>::convert_float() const - { - intvec_t r(_mm512_undefined_pd()); - for (int n=0; n<size; ++n) { - r.set_elt(n, floatprops::convert_float((*this)[n])); - } - return r; } - - inline intvec<double,8> intvec<double,8>::abs() const - { - return MF::vml_abs(*this); - } - - inline intvec<double,8> intvec<double,8>::bitifthen(intvec_t x, - intvec_t y) const - { - return MF::vml_bitifthen(*this, x, y); - } - - inline boolvec<double,8> intvec<double,8>::isignbit() const - { - return MF::vml_isignbit(*this); - } - - inline intvec<double,8> intvec<double,8>::max(intvec_t x) const - { - return MF::vml_max(*this, x); - } - - inline intvec<double,8> intvec<double,8>::min(intvec_t x) const - { - return MF::vml_min(*this, x); - } - - inline intvec<double,8> intvec<double,8>::rotate(int_t n) const - { - return MF::vml_rotate(*this, n); - } - - inline intvec<double,8> intvec<double,8>::rotate(intvec_t n) const - { - return MF::vml_rotate(*this, n); - } - + boolvec_t isnormal() const { return MF::vml_isnormal(*this); } + realvec ldexp(int_t n) const { return MF::vml_ldexp(*this, n); } + realvec ldexp(intvec_t n) const { return MF::vml_ldexp(*this, n); } + realvec log() const { return MF::vml_log(*this); } + realvec log10() const { return MF::vml_log10(*this); } + realvec log1p() const { return MF::vml_log1p(*this); } + realvec log2() const { return MF::vml_log2(*this); } + realvec_t mad(realvec_t y, realvec_t z) const { + return _mm512_fmadd_pd(v, x.v, y.v); + } + realvec nextafter(realvec y) const { return MF::vml_nextafter(*this, y); } + realvec pow(realvec y) const { return MF::vml_pow(*this, y); } + realvec rcp() const { return _mm512_div_pd(_mm512_set1_pd(1.0), v); } + realvec remainder(realvec y) const { return MF::vml_remainder(*this, y); } + realvec rint() const { return _mm512_round_pd(v, _MM_FROUND_TO_NEAREST_INT); } + realvec round() const { return MF::vml_round(*this); } + realvec rsqrt() const { return MF::vml_rsqrt(*this); } + boolvec_t signbit() const { return as_int().signbit(); } + realvec sin() const { return MF::vml_sin(*this); } + realvec sinh() const { return MF::vml_sinh(*this); } + realvec sqrt() const { return _mm512_sqrt_pd(v); } + realvec tan() const { return MF::vml_tan(*this); } + realvec tanh() const { return MF::vml_tanh(*this); } + realvec trunc() const { return _mm512_round_pd(v, _MM_FROUND_TO_ZERO); } +}; + +// boolvec definitions + +inline intvec<double, 4> boolvec<double, 4>::as_int() const { + return _mm512_castpd_si512(v); +} + +inline intvec<double, 4> boolvec<double, 4>::convert_int() const { + return ifthen(v, IV(I(1)), IV(I(0))); +} + +inline boolvec<double, 4> boolvec<double, 4>::ifthen(boolvec_t x, + boolvec_t y) const { + return (v & x.v) | (~v & y.v); +} + +inline intvec<double, 4> boolvec<double, 4>::ifthen(intvec_t x, + intvec_t y) const { + return _mm512_blend_epi64(v, y.v, x.v) +} + +inline realvec<double, 4> boolvec<double, 4>::ifthen(realvec_t x, + realvec_t y) const { + return _mm512_blend_pd(v, y.v, x.v) +} + +// intvec definitions + +inline realvec<double, 4> intvec<double, 4>::as_float() const { + return _mm512_castsi512_pd(v); +} + +inline realvec<double, 4> intvec<double, 4>::convert_float() const { + intvec_t r(_mm512_undefined_pd()); + for (int n = 0; n < size; ++n) { + r.set_elt(n, floatprops::convert_float((*this)[n])); + } + return r; +} + +inline intvec<double, 8> intvec<double, 8>::abs() const { + return MF::vml_abs(*this); +} + +inline intvec<double, 8> intvec<double, 8>::bitifthen(intvec_t x, + intvec_t y) const { + return MF::vml_bitifthen(*this, x, y); +} + +inline boolvec<double, 8> intvec<double, 8>::isignbit() const { + return MF::vml_isignbit(*this); +} + +inline intvec<double, 8> intvec<double, 8>::max(intvec_t x) const { + return MF::vml_max(*this, x); +} + +inline intvec<double, 8> intvec<double, 8>::min(intvec_t x) const { + return MF::vml_min(*this, x); +} + +inline intvec<double, 8> intvec<double, 8>::rotate(int_t n) const { + return MF::vml_rotate(*this, n); +} + +inline intvec<double, 8> intvec<double, 8>::rotate(intvec_t n) const { + return MF::vml_rotate(*this, n); +} + } // namespace vecmathlib -#endif // #ifndef VEC_MIC_DOUBLE8_H +#endif // #ifndef VEC_MIC_DOUBLE8_H |
