// -*-C++-*- #ifndef VEC_AVX_FLOAT8_H #define VEC_AVX_FLOAT8_H #include "floatprops.h" #include "mathfuncs.h" #include "vec_base.h" #include // AVX intrinsics #include namespace vecmathlib { #define VECMATHLIB_HAVE_VEC_FLOAT_8 template<> struct boolvec; template<> struct intvec; template<> struct realvec; template<> struct boolvec: floatprops { static int const size = 8; typedef bool scalar_t; typedef __m256 bvector_t; static int const alignment = sizeof(bvector_t); static_assert(size * sizeof(real_t) == sizeof(bvector_t), "vector size is wrong"); private: // true values have the sign bit set, false values have it unset static uint_t from_bool(bool a) { return - uint_t(a); } static bool to_bool(uint_t a) { return int_t(a) < int_t(0); } public: typedef boolvec boolvec_t; typedef intvec intvec_t; typedef realvec realvec_t; // Short names for type casts typedef real_t R; typedef int_t I; typedef uint_t U; typedef realvec_t RV; typedef intvec_t IV; typedef boolvec_t BV; typedef floatprops FP; typedef mathfuncs MF; bvector_t v; boolvec() {} // Can't have a non-trivial copy constructor; if so, objects won't // be passed in registers // boolvec(boolvec const& x): v(x.v) {} // boolvec& operator=(boolvec const& x) { return v=x.v, *this; } boolvec(bvector_t x): v(x) {} boolvec(bool a): v(_mm256_castsi256_ps(_mm256_set1_epi32(from_bool(a)))) {} boolvec(bool const* as): v(_mm256_castsi256_ps(_mm256_set_epi32(from_bool(as[7]), from_bool(as[6]), from_bool(as[5]), from_bool(as[4]), from_bool(as[3]), from_bool(as[2]), from_bool(as[1]), from_bool(as[0])))) {} operator bvector_t() const { return v; } bool operator[](int n) const { return to_bool(vecmathlib::get_elt(v, n)); } boolvec_t& set_elt(int n, bool a) { return vecmathlib::set_elt(v, n, from_bool(a)), *this; } intvec_t as_int() const; // defined after intvec intvec_t convert_int() const; // defined after intvec boolvec_t operator!() const { return _mm256_xor_ps(boolvec(true), v); } boolvec_t operator&&(boolvec_t x) const { return _mm256_and_ps(v, x.v); } boolvec_t operator||(boolvec_t x) const { return _mm256_or_ps(v, x.v); } boolvec_t operator==(boolvec_t x) const { return !(*this!=x); } boolvec_t operator!=(boolvec_t x) const { return _mm256_xor_ps(v, x.v); } bool all() const { // return // (*this)[0] && (*this)[1] && (*this)[2] && (*this)[3] && // (*this)[4] && (*this)[5] && (*this)[6] && (*this)[7]; return ! (! *this).any(); } bool any() const { // return // (*this)[0] || (*this)[1] || (*this)[2] || (*this)[3] || // (*this)[4] || (*this)[5] || (*this)[6] || (*this)[7]; return ! bool(_mm256_testz_ps(v, v)); } // ifthen(condition, then-value, else-value) boolvec_t ifthen(boolvec_t x, boolvec_t y) const; intvec_t ifthen(intvec_t x, intvec_t y) const; // defined after intvec realvec_t ifthen(realvec_t x, realvec_t y) const; // defined after realvec }; template<> struct intvec: floatprops { static int const size = 8; typedef int_t scalar_t; typedef __m256i ivector_t; static int const alignment = sizeof(ivector_t); static_assert(size * sizeof(real_t) == sizeof(ivector_t), "vector size is wrong"); typedef boolvec boolvec_t; typedef intvec intvec_t; typedef realvec realvec_t; // Short names for type casts typedef real_t R; typedef int_t I; typedef uint_t U; typedef realvec_t RV; typedef intvec_t IV; typedef boolvec_t BV; typedef floatprops FP; typedef mathfuncs MF; ivector_t v; intvec() {} // Can't have a non-trivial copy constructor; if so, objects won't // be passed in registers // intvec(intvec const& x): v(x.v) {} // intvec& operator=(intvec const& x) { return v=x.v, *this; } intvec(ivector_t x): v(x) {} intvec(int_t a): v(_mm256_set1_epi32(a)) {} intvec(int_t const* as): v(_mm256_set_epi32(as[7], as[6], as[5], as[4], as[3], as[2], as[1], as[0])) {} static intvec_t iota() { return _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); } operator ivector_t() const { return v; } int_t operator[](int n) const { return vecmathlib::get_elt(v, n); } intvec_t& set_elt(int n, int_t a) { return vecmathlib::set_elt(v, n, a), *this; } boolvec_t as_bool() const { return _mm256_castsi256_ps(v); } boolvec_t convert_bool() const { // Result: convert_bool(0)=false, convert_bool(else)=true // There is no intrinsic to compare with zero. Instead, we check // whether x is positive and x-1 is negative. intvec_t x = *this; // We know that boolvec_t values depend only on the sign bit // return (~(x-1) | x).as_bool(); // return x.as_bool() || !(x-1).as_bool(); return x.as_bool() || (x + (FP::signbit_mask - 1)).as_bool(); } realvec_t as_float() const; // defined after realvec realvec_t convert_float() const; // defined after realvec // Note: not all arithmetic operations are supported! intvec_t operator+() const { return *this; } intvec_t operator-() const { return IV(0) - *this; } intvec_t operator+(intvec_t x) const { __m128i vlo = _mm256_castsi256_si128(v); __m128i vhi = _mm256_extractf128_si256(v, 1); __m128i xvlo = _mm256_castsi256_si128(x.v); __m128i xvhi = _mm256_extractf128_si256(x.v, 1); vlo = _mm_add_epi32(vlo, xvlo); vhi = _mm_add_epi32(vhi, xvhi); return _mm256_insertf128_si256(_mm256_castsi128_si256(vlo), vhi, 1); } intvec_t operator-(intvec_t x) const { __m128i vlo = _mm256_castsi256_si128(v); __m128i vhi = _mm256_extractf128_si256(v, 1); __m128i xvlo = _mm256_castsi256_si128(x.v); __m128i xvhi = _mm256_extractf128_si256(x.v, 1); vlo = _mm_sub_epi32(vlo, xvlo); vhi = _mm_sub_epi32(vhi, xvhi); return _mm256_insertf128_si256(_mm256_castsi128_si256(vlo), vhi, 1); } intvec_t& operator+=(intvec_t const& x) { return *this=*this+x; } intvec_t& operator-=(intvec_t const& x) { return *this=*this-x; } intvec_t operator~() const { return IV(~U(0)) ^ *this; } intvec_t operator&(intvec_t x) const { return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(v), _mm256_castsi256_ps(x.v))); } intvec_t operator|(intvec_t x) const { return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(v), _mm256_castsi256_ps(x.v))); } intvec_t operator^(intvec_t x) const { return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(v), _mm256_castsi256_ps(x.v))); } intvec_t& operator&=(intvec_t const& x) { return *this=*this&x; } intvec_t& operator|=(intvec_t const& x) { return *this=*this|x; } intvec_t& operator^=(intvec_t const& x) { return *this=*this^x; } intvec_t bitifthen(intvec_t x, intvec_t y) const; intvec_t lsr(int_t n) const { __m128i vlo = _mm256_castsi256_si128(v); __m128i vhi = _mm256_extractf128_si256(v, 1); vlo = _mm_srli_epi32(vlo, n); vhi = _mm_srli_epi32(vhi, n); return _mm256_insertf128_si256(_mm256_castsi128_si256(vlo), vhi, 1); } intvec_t rotate(int_t n) const; intvec_t operator>>(int_t n) const { __m128i vlo = _mm256_castsi256_si128(v); __m128i vhi = _mm256_extractf128_si256(v, 1); vlo = _mm_srai_epi32(vlo, n); vhi = _mm_srai_epi32(vhi, n); return _mm256_insertf128_si256(_mm256_castsi128_si256(vlo), vhi, 1); } intvec_t operator<<(int_t n) const { __m128i vlo = _mm256_castsi256_si128(v); __m128i vhi = _mm256_extractf128_si256(v, 1); vlo = _mm_slli_epi32(vlo, n); vhi = _mm_slli_epi32(vhi, n); return _mm256_insertf128_si256(_mm256_castsi128_si256(vlo), vhi, 1); } intvec_t& operator>>=(int_t n) { return *this=*this>>n; } intvec_t& operator<<=(int_t n) { return *this=*this<> U(n[i])); } return r; } intvec_t rotate(intvec_t n) const; intvec_t operator>>(intvec_t n) const { intvec_t r; for (int i=0; i> n[i]); } return r; } intvec_t operator<<(intvec_t n) const { intvec_t r; for (int i=0; i>=(intvec_t n) { return *this=*this>>n; } intvec_t& operator<<=(intvec_t n) { return *this=*this< x); } boolvec_t operator>(intvec_t const& x) const { return x < *this; } boolvec_t operator>=(intvec_t const& x) const { return ! (*this < x); } intvec_t abs() const; boolvec_t isignbit() const { return as_bool(); } intvec_t max(intvec_t x) const; intvec_t min(intvec_t x) const; }; template<> struct realvec: floatprops { static int const size = 8; typedef real_t scalar_t; typedef __m256 vector_t; static int const alignment = sizeof(vector_t); static char const* name() { return ""; } void barrier() { __asm__("": "+x"(v)); } static_assert(size * sizeof(real_t) == sizeof(vector_t), "vector size is wrong"); typedef boolvec boolvec_t; typedef intvec intvec_t; typedef realvec realvec_t; // Short names for type casts typedef real_t R; typedef int_t I; typedef uint_t U; typedef realvec_t RV; typedef intvec_t IV; typedef boolvec_t BV; typedef floatprops FP; typedef mathfuncs MF; vector_t v; realvec() {} // Can't have a non-trivial copy constructor; if so, objects won't // be passed in registers // realvec(realvec const& x): v(x.v) {} // realvec& operator=(realvec const& x) { return v=x.v, *this; } realvec(vector_t x): v(x) {} realvec(real_t a): v(_mm256_set1_ps(a)) {} realvec(real_t const* as): v(_mm256_set_ps(as[7], as[6], as[5], as[4], as[3], as[2], as[1], as[0])) {} operator vector_t() const { return v; } real_t operator[](int n) const { return vecmathlib::get_elt(v, n); } realvec_t& set_elt(int n, real_t a) { return vecmathlib::set_elt(v, n, a), *this; } typedef vecmathlib::mask_t mask_t; static realvec_t loada(real_t const* p) { VML_ASSERT(intptr_t(p) % alignment == 0); return _mm256_load_ps(p); } static realvec_t loadu(real_t const* p) { return _mm256_loadu_ps(p); } static realvec_t loadu(real_t const* p, std::ptrdiff_t ioff) { VML_ASSERT(intptr_t(p) % alignment == 0); if (ioff % realvec::size == 0) return loada(p+ioff); return loadu(p+ioff); } realvec_t loada(real_t const* p, mask_t const& m) const { VML_ASSERT(intptr_t(p) % alignment == 0); if (__builtin_expect(all(m.m), true)) { return loada(p); } else { return m.m.ifthen(loada(p), *this); } } realvec_t loadu(real_t const* p, mask_t const& m) const { if (__builtin_expect(m.all_m, true)) { return loadu(p); } else { return m.m.ifthen(loadu(p), *this); } } realvec_t loadu(real_t const* p, std::ptrdiff_t ioff, mask_t const& m) const { VML_ASSERT(intptr_t(p) % alignment == 0); if (ioff % realvec::size == 0) return loada(p+ioff, m); return loadu(p+ioff, m); } void storea(real_t* p) const { VML_ASSERT(intptr_t(p) % alignment == 0); _mm256_store_ps(p, v); } void storeu(real_t* p) const { return _mm256_storeu_ps(p, v); } void storeu(real_t* p, std::ptrdiff_t ioff) const { VML_ASSERT(intptr_t(p) % alignment == 0); if (ioff % realvec::size == 0) return storea(p+ioff); storeu(p+ioff); } void storea(real_t* p, mask_t const& m) const { VML_ASSERT(intptr_t(p) % alignment == 0); if (__builtin_expect(m.all_m, true)) { storea(p); } else { _mm256_maskstore_ps(p, m.m.as_int(), v); } } void storeu(real_t* p, mask_t const& m) const { if (__builtin_expect(m.all_m, true)) { storeu(p); } else { // TODO: this is expensive for (int n=0; n(realvec_t const& x) const { return _mm256_cmp_ps(v, x.v, _CMP_GT_OQ); } boolvec_t operator>=(realvec_t const& x) const { return _mm256_cmp_ps(v, x.v, _CMP_GE_OQ); } realvec_t acos() const { return MF::vml_acos(*this); } realvec_t acosh() const { return MF::vml_acosh(*this); } realvec_t asin() const { return MF::vml_asin(*this); } realvec_t asinh() const { return MF::vml_asinh(*this); } realvec_t atan() const { return MF::vml_atan(*this); } realvec_t atan2(realvec_t y) const { return MF::vml_atan2(*this, y); } realvec_t atanh() const { return MF::vml_atanh(*this); } realvec_t cbrt() const { return MF::vml_cbrt(*this); } realvec_t ceil() const { return _mm256_ceil_ps(v); } realvec_t copysign(realvec_t y) const { return MF::vml_copysign(*this, y); } realvec_t cos() const { return MF::vml_cos(*this); } realvec_t cosh() const { return MF::vml_cosh(*this); } realvec_t exp() const { return MF::vml_exp(*this); } realvec_t exp10() const { return MF::vml_exp10(*this); } realvec_t exp2() const { return MF::vml_exp2(*this); } realvec_t expm1() const { return MF::vml_expm1(*this); } realvec_t fabs() const { return MF::vml_fabs(*this); } realvec_t fdim(realvec_t y) const { return MF::vml_fdim(*this, y); } realvec_t floor() const { return _mm256_floor_ps(v); } realvec_t fma(realvec_t y, realvec_t z) const { return MF::vml_fma(*this, y, z); } realvec_t fmax(realvec_t y) const { return _mm256_max_ps(v, y.v); } realvec_t fmin(realvec_t y) const { return _mm256_min_ps(v, y.v); } realvec_t fmod(realvec_t y) const { return MF::vml_fmod(*this, y); } realvec_t frexp(intvec_t* r) const { return MF::vml_frexp(*this, r); } realvec_t hypot(realvec_t y) const { return MF::vml_hypot(*this, y); } intvec_t ilogb() const { return MF::vml_ilogb(*this); } boolvec_t isfinite() const { return MF::vml_isfinite(*this); } boolvec_t isinf() const { return MF::vml_isinf(*this); } boolvec_t isnan() const { #ifdef VML_HAVE_NAN return _mm256_cmp_ps(v, v, _CMP_UNORD_Q); #else return BV(false); #endif } boolvec_t isnormal() const { return MF::vml_isnormal(*this); } realvec_t ldexp(int_t n) const { return MF::vml_ldexp(*this, n); } realvec_t ldexp(intvec_t n) const { return MF::vml_ldexp(*this, n); } realvec_t log() const { return MF::vml_log(*this); } realvec_t log10() const { return MF::vml_log10(*this); } realvec_t log1p() const { return MF::vml_log1p(*this); } realvec_t log2() const { return MF::vml_log2(*this); } realvec_t mad(realvec_t y, realvec_t z) const { return MF::vml_mad(*this, y, z); } realvec_t nextafter(realvec_t y) const { return MF::vml_nextafter(*this, y); } realvec_t pow(realvec_t y) const { return MF::vml_pow(*this, y); } realvec_t rcp() const { realvec_t x = *this; realvec_t r = _mm256_rcp_ps(x); // this is only an approximation r *= RV(2.0) - r*x; // one Newton iteration (see vml_rcp) return r; } realvec_t remainder(realvec_t y) const { return MF::vml_remainder(*this, y); } realvec_t rint() const { return _mm256_round_ps(v, _MM_FROUND_TO_NEAREST_INT); } realvec_t round() const { return MF::vml_round(*this); } realvec_t rsqrt() const { realvec_t x = *this; realvec_t r = _mm256_rsqrt_ps(x); // this is only an approximation r *= RV(1.5) - RV(0.5)*x * r*r; // one Newton iteration (see vml_rsqrt) return r; } boolvec_t signbit() const { return v; } realvec_t sin() const { return MF::vml_sin(*this); } realvec_t sinh() const { return MF::vml_sinh(*this); } realvec_t sqrt() const { return _mm256_sqrt_ps(v); } realvec_t tan() const { return MF::vml_tan(*this); } realvec_t tanh() const { return MF::vml_tanh(*this); } realvec_t trunc() const { return _mm256_round_ps(v, _MM_FROUND_TO_ZERO); } }; // boolvec definitions inline intvec boolvec::as_int() const { return _mm256_castps_si256(v); } inline intvec boolvec::convert_int() const { return lsr(as_int(), bits-1); } inline boolvec boolvec::ifthen(boolvec_t x, boolvec_t y) const { return ifthen(x.as_int(), y.as_int()).as_bool(); } inline intvec boolvec::ifthen(intvec_t x, intvec_t y) const { return ifthen(x.as_float(), y.as_float()).as_int(); } inline realvec boolvec::ifthen(realvec_t x, realvec_t y) const { return _mm256_blendv_ps(y.v, x.v, v); } // intvec definitions inline intvec intvec::abs() const { return MF::vml_abs(*this); } inline realvec intvec::as_float() const { return _mm256_castsi256_ps(v); } inline intvec intvec::bitifthen(intvec_t x, intvec_t y) const { return MF::vml_bitifthen(*this, x, y); } inline intvec intvec::clz() const { return MF::vml_clz(*this); } inline realvec intvec::convert_float() const { return _mm256_cvtepi32_ps(v); } inline intvec intvec::max(intvec_t x) const { return MF::vml_max(*this, x); } inline intvec intvec::min(intvec_t x) const { return MF::vml_min(*this, x); } inline intvec intvec::popcount() const { return MF::vml_popcount(*this); } inline intvec intvec::rotate(int_t n) const { return MF::vml_rotate(*this, n); } inline intvec intvec::rotate(intvec_t n) const { return MF::vml_rotate(*this, n); } } // namespace vecmathlib #endif // #ifndef VEC_AVX_FLOAT8_H