path: root/vec_mic_double8.h

// -*-C++-*-

#ifndef VEC_MIC_DOUBLE8_H
#define VEC_MIC_DOUBLE8_H

#include "floatprops.h"
#include "mathfuncs.h"
#include "vec_base.h"

#include <cmath>

// MIC intrinsics
#include <immintrin.h>



namespace vecmathlib {
  
#define VECMATHLIB_HAVE_VEC_DOUBLE_8
  template<> struct boolvec<double,8>;
  template<> struct intvec<double,8>;
  template<> struct realvec<double,8>;
  
  
  
  template<>
  struct boolvec<double,8>: floatprops<double>
  {
    static const int size = 8;
    typedef bool scalar_t;
    typedef __mask8 bvector_t;
    static const int alignment = sizeof(bvector_t);
    
    // static_assert(size * sizeof(real_t) == sizeof(bvector_t),
    //               "vector size is wrong");
    
    typedef boolvec boolvec_t;
    typedef intvec<real_t, size> intvec_t;
    typedef realvec<real_t, size> realvec_t;
    
    // Short names for type casts
    typedef real_t R;
    typedef int_t I;
    typedef uint_t U;
    typedef realvec_t RV;
    typedef intvec_t IV;
    typedef boolvec_t BV;
    typedef floatprops<real_t> FP;
    typedef mathfuncs<realvec_t> MF;
    
    
    
    bvector_t v;
    
    boolvec() {}
    // Can't have a non-trivial copy constructor; if so, objects won't
    // be passed in registers
    // boolvec(const boolvec& x): v(x.v) {}
    // boolvec& operator=(const boolvec& x) { return v=x.v, *this; }
    boolvec(bvector_t x): v(x) {}
    boolvec(bool a): v(- bvector_t(a)) {}
    boolvec(const bool* as):
      v((bvector_t(as[0]) << 0) |
        (bvector_t(as[1]) << 1) |
        (bvector_t(as[2]) << 2) |
        (bvector_t(as[3]) << 3) |
        (bvector_t(as[4]) << 4) |
        (bvector_t(as[5]) << 5) |
        (bvector_t(as[6]) << 6) |
        (bvector_t(as[7]) << 7))
    {}
    
    operator bvector_t() const { return v; }
    bool operator[](int n) const
    {
      return (v >> n) & 1;
    }
    boolvec& set_elt(int n, bool a)
    {
      v &= ~ (bvector_t(1) << n);
      v |= bvector_t(a) << n;
      return *this;
    }
    
    
    
    intvec_t as_int() const;      // defined after intvec
    intvec_t convert_int() const; // defined after intvec
    
    
    
    boolvec operator!() const { return _mm512_knot(v); }
    
    boolvec operator&&(boolvec x) const { return _mm512_kand(v, x.v); }
    boolvec operator||(boolvec x) const { return _mm512_kor(v, x.v); }
    boolvec operator==(boolvec x) const { return _mm512_kxnor(v, x.v); }
    boolvec operator!=(boolvec x) const { return _mm512_kxor(v, x.v); }
    
    bool all() const { return _mm512_kortestc(v, v); }
    bool any() const { return ! _mm512_kortestz(v, v); }
    
    
    
    // ifthen(condition, then-value, else-value)
    boolvec_t ifthen(boolvec_t x, boolvec_t y) const;
    intvec_t ifthen(intvec_t x, intvec_t y) const; // defined after intvec
    realvec_t ifthen(realvec_t x, realvec_t y) const; // defined after realvec
  };
  
  
  
  template<>
  struct intvec<double,8>: floatprops<double>
  {
    static const int size = 8;
    typedef int_t scalar_t;
    typedef __m512i ivector_t;
    static const int alignment = sizeof(ivector_t);
    
    static_assert(size * sizeof(real_t) == sizeof(ivector_t),
                  "vector size is wrong");
    
    typedef boolvec<real_t, size> boolvec_t;
    typedef intvec intvec_t;
    typedef realvec<real_t, size> realvec_t;
    
    // Short names for type casts
    typedef real_t R;
    typedef int_t I;
    typedef uint_t U;
    typedef realvec_t RV;
    typedef intvec_t IV;
    typedef boolvec_t BV;
    typedef floatprops<real_t> FP;
    typedef mathfuncs<realvec_t> MF;
    
    
    
    ivector_t v;
    
    intvec() {}
    // Can't have a non-trivial copy constructor; if so, objects won't
    // be passed in registers
    // intvec(const intvec& x): v(x.v) {}
    // intvec& operator=(const intvec& x) { return v=x.v, *this; }
    intvec(ivector_t x): v(x) {}
    intvec(int_t a): v(_mm512_set1_epi64(a)) {}
    intvec(const int_t* as)
    {
      v = _mm512_undefined_epi32();
      // v = _mm512_loadunpacklo_epi32(v, as);
      // v = _mm512_loadunpackhi_epi32(v, as+8);
      for (int n=0; n<size; ++n) set_elt(n, as[n]);
    }
    static intvec iota()
    {
      intvec r;
      for (int n=0; n<size; ++n) r.set_elt(n, n);
      return r;
    }
    
    operator ivector_t() const { return v; }
    int_t operator[](int n) const
    {
      return vecmathlib::get_elt<IV,ivector_t,int_t>(v, n);
    }
    intvec_t& set_elt(int n, int_t a)
    {
      return vecmathlib::set_elt<IV,ivector_t,int_t>(v, n, a), *this;
    }
    
    
    
  private:
    static __mmask8 mask16tomask8(__mmask16 m16)
    {
      // combine 01
      m16 = ((m16 >> 1) | m16) & 0b0011001100110011;
      // combine 0123
      m16 = ((m16 >> 2) | m16) & 0b0000111100001111;
      // combine 01234567
      m16 = ((m16 >> 4) | m16) & 0b0000000011111111;
      return m16;
    }
  public:
    boolvec_t as_bool() const { return convert_bool(); }
    boolvec_t convert_bool() const
    {
      // Result: convert_bool(0)=false, convert_bool(else)=true
      __mmask16 r16 = _mm512_test_epi32_mask(v, v);
      return mask16tomask8(r16);
    }
    realvec_t as_float() const;      // defined after realvec
    realvec_t convert_float() const; // defined after realvec
    
    
    
    // Note: not all arithmetic operations are supported!
    
    intvec operator+() const { return *this; }
    intvec operator-() const { return IV(I(0)) - *this; }
    intvec operator+(intvec x) const { return _mm512_add_epi64(v, x.v); }
    intvec operator-(intvec x) const { return _mm512_sub_epi64(v, x.v); }
    
    intvec& operator+=(const intvec& x) { return *this=*this+x; }
    intvec& operator-=(const intvec& x) { return *this=*this-x; }
    
    
    
    intvec operator~() const { return IV(~U(0)) ^ *this; }
    intvec operator&(intvec x) const { return _mm512_and_epi64(v, x.v); }
    intvec operator|(intvec x) const { return _mm512_or_epi64(v, x.v); }
    intvec operator^(intvec x) const { return _mm512_xor_epi64(v, x.v); }
    
    intvec& operator&=(const intvec& x) { return *this=*this&x; }
    intvec& operator|=(const intvec& x) { return *this=*this|x; }
    intvec& operator^=(const intvec& x) { return *this=*this^x; }
    
    
    
    intvec lsr(int_t n) const
    {
      if (n < 32) {
        __m512i vlo = _mm512_srli_epi32(v, n);
        __m512i vhi = _mm512_slli_epi32(v, 32-n);
        vhi = _mm512_swizzle_epi32(vhi, _MM_SWIZ_REG_CDAB);
        return _mm512_mask_or_epi32(vlo, 0xb0101010101010101, vhi, vlo);
      } else {
        __m512i vlo = _mm512_srli_epi32(v, n-32);
        __m512i vhi = _mm512_setzero_epi32();
        return _mm512_mask_swizzle_epi32(vhi, 0xb0101010101010101, vlo);
      }
    }
    intvec operator>>(int_t n) const
    {
      if (n < 32) {
        __mm512i vlo = _mm512_srai_epi32(v, n);
        __mm512i vlo0 = _mm512_srli_epi32(v, n);
        __mm512i vhi = _mm512_slli_epi32(v, 32-n);
        vhi = _mm512_swizzle_epi32(vhi, _MM_SWIZ_REG_CDAB);
        return _mm512_mask_or_epi32(vlo, 0xb0101010101010101, vhi, vlo0);
      } else {
        __m512i vlo = _mm512_srai_epi32(v, n-32);
        __m512i vhi = _mm512_srai_epi32(v, 31);
        return _mm512_mask_swizzle_epi32(vhi, 0xb0101010101010101, vlo);
      }
    }
    intvec operator<<(int_t n) const
    {
      if (n < 32) {
        __m512i vlo = _mm512_srli_epi32(v, n);
        __m512i vhi = _mm512_slli_epi32(v, 32-n);
        vlo = _mm512_swizzle_epi32(vlo, _MM_SWIZ_REG_CDAB);
        return _mm512_mask_or_epi32(vhi, 0xb1010101010101010, vhi, vlo);
      } else {
        __m512i vlo = _mm512_setzero_epi32();
        __m512i vhi = _mm512_slli_epi32(v, n-32);
        return _mm512_mask_swizzle_epi32(vhi, 0xb1010101010101010, vlo);
      }
    }
    intvec& operator>>=(int_t n) { return *this=*this>>n; }
    intvec& operator<<=(int_t n) { return *this=*this<<n; }
    
    intvec lsr(intvec n) const
    {
      // TODO: improve this
      intvec r;
      for (int i=0; i<size; ++i) {
        r.set_elt(i, U((*this)[i]) >> U(n[i]));
      }
      return r;
    }
    intvec operator>>(intvec n) const
    {
      // TODO: improve this
      intvec r;
      for (int i=0; i<size; ++i) {
        r.set_elt(i, (*this)[i] >> n[i]);
      }
      return r;
    }
    intvec operator<<(intvec n) const
    {
      // TODO: improve this
      intvec r;
      for (int i=0; i<size; ++i) {
        r.set_elt(i, (*this)[i] << n[i]);
      }
      return r;
    }
    intvec& operator>>=(intvec n) { return *this=*this>>n; }
    intvec& operator<<=(intvec n) { return *this=*this<<n; }
    
    
    
    boolvec_t signbit() const
    {
      return *this < IV(I(0));
    }
    
    boolvec_t operator==(const intvec& x) const
    {
      return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_EQ));
    }
    boolvec_t operator!=(const intvec& x) const
    {
      return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_NE));
    }
    boolvec_t operator<(const intvec& x) const
    {
      return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_LT));
    }
    boolvec_t operator<=(const intvec& x) const
    {
      return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_LE));
    }
    boolvec_t operator>(const intvec& x) const
    {
      return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_GT));
    }
    boolvec_t operator>=(const intvec& x) const
    {
      return mask16tomask8(_mm512_cmp_epi32_mask(v, x.v, _MM_CMPINT_GE));
    }
  };
  
  
  
  template<>
  struct realvec<double,8>: floatprops<double>
  {
    static const int size = 8;
    typedef real_t scalar_t;
    typedef __m512d vector_t;
    static const int alignment = sizeof(vector_t);
    
    static const char* name() { return "<MIC:8*double>"; }
    void barrier() { __asm__("": "+x"(v)); }
    
    static_assert(size * sizeof(real_t) == sizeof(vector_t),
                  "vector size is wrong");
    
    typedef boolvec<real_t, size> boolvec_t;
    typedef intvec<real_t, size> intvec_t;
    typedef realvec realvec_t;
    
    // Short names for type casts
    typedef real_t R;
    typedef int_t I;
    typedef uint_t U;
    typedef realvec_t RV;
    typedef intvec_t IV;
    typedef boolvec_t BV;
    typedef floatprops<real_t> FP;
    typedef mathfuncs<realvec_t> MF;
    
    
    
    vector_t v;
    
    realvec() {}
    // Can't have a non-trivial copy constructor; if so, objects won't
    // be passed in registers
    // realvec(const realvec& x): v(x.v) {}
    // realvec& operator=(const realvec& x) { return v=x.v, *this; }
    realvec(vector_t x): v(x) {}
    realvec(real_t a): v(_mm512_set1_pd(a)) {}
    realvec(const real_t* as)
    {
      v = _mm512_undefined_pd();
      // v = _mm512_loadunpacklo_pd(v, as);
      // v = _mm512_loadunpackhi_pd(v, as+8);
      for (int n=0; n<size; ++n) set_elt(n, as[n]);
    }
    
    operator vector_t() const { return v; }
    real_t operator[](int n) const
    {
      return vecmathlib::get_elt<RV,vector_t,real_t>(v, n);
    }
    realvec_t& set_elt(int n, real_t a)
    {
      return vecmathlib::set_elt<RV,vector_t,real_t>(v, n, a), *this;
    }
    
    
    
    typedef vecmathlib::mask_t<realvec_t> mask_t;
    
    static realvec_t loada(const real_t* p)
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      return _mm512_load_pd(p);
    }
    static realvec_t loadu(const real_t* p)
    {
      realvec_t r(_mm512_undefined_pd());
      r.v = _mm512_loadunpacklo_pd(r.v, p);
      r.v = _mm512_loadunpackhi_pd(r.v, p+8);
      return r.v;
    }
    static realvec_t loadu(const real_t* p, std::ptrdiff_t ioff)
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      if (ioff % realvec::size == 0) return loada(p+ioff);
      return loadu(p+ioff);
    }
    realvec_t loada(const real_t* p, const mask_t& m) const
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      return _mm512_mask_load_pd(v, m.m.v, p);
    }
    realvec_t loadu(const real_t* p, const mask_t& m) const
    {
      if (__builtin_expect(m.all_m, true)) {
        return loadu(p);
      } else {
        return m.m.ifthen(loadu(p), *this);
      }
    }
    realvec_t loadu(const real_t* p, std::ptrdiff_t ioff, const mask_t& m) const
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      if (ioff % realvec::size == 0) return loada(p+ioff, m);
      return loadu(p+ioff, m);
    }
    
    void storea(real_t* p) const
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      _mm512_store_pd(p, v);
    }
    void storeu(real_t* p) const
    {
      _mm512_packstorelo_pd(p, v);
      _mm512_packstorehi_pd(p+8, v);
    }
    void storeu(real_t* p, std::ptrdiff_t ioff) const
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      if (ioff % realvec::size == 0) return storea(p+ioff);
      storeu(p+ioff);
    }
    void storea(real_t* p, const mask_t& m) const
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      _mm512_mask_store_pd(p, m.m.v, v);
    }
    void storeu(real_t* p, const mask_t& m) const
    {
      if (__builtin_expect(m.all_m, true)) {
        storeu(p);
      } else {
        for (int n=0; n<size; ++n) {
          if (m.m[n]) p[n] = (*this)[n];
        }
      }
    }
    void storeu(real_t* p, std::ptrdiff_t ioff, const mask_t& m) const
    {
      VML_ASSERT(intptr_t(p) % alignment == 0);
      if (ioff % realvec::size == 0) return storea(p+ioff, m);
      storeu(p+ioff, m);
    }
    
    
    
    intvec_t as_int() const { return _mm512_castpd_si512(v); }
    intvec_t convert_int() const
    {
      intvec_t r(_mm512_undefined_epi32());
      for (int n=0; n<size; ++n) {
        r.set_elt(n, floatprops::convert_int((*this)[n]));
      }
      return r;
    }
    
    
    
    realvec operator+() const { return *this; }
    realvec operator-() const { return RV(0.0) - *this; }
    
    realvec operator+(realvec x) const { return _mm512_add_pd(v, x.v); }
    realvec operator-(realvec x) const { return _mm512_sub_pd(v, x.v); }
    realvec operator*(realvec x) const { return _mm512_mul_pd(v, x.v); }
    realvec operator/(realvec x) const { return _mm512_div_pd(v, x.v); }
    
    realvec& operator+=(const realvec& x) { return *this=*this+x; }
    realvec& operator-=(const realvec& x) { return *this=*this-x; }
    realvec& operator*=(const realvec& x) { return *this=*this*x; }
    realvec& operator/=(const realvec& x) { return *this=*this/x; }
    
    real_t maxval() const { returm _mm512_reduce_gmax_pd(v); }
    real_t minval() const { returm _mm512_reduce_gmin_pd(v); }
    real_t prod() const { returm _mm512_reduce_mul_pd(v); }
    real_t sum() const { returm _mm512_reduce_add_pd(v); }
    
    
    
    boolvec_t operator==(const realvec& x) const
    {
      return _mm512_cmp_pd(v, x.v, _CMP_EQ_OQ);
    }
    boolvec_t operator!=(const realvec& x) const
    {
      return _mm512_cmp_pd(v, x.v, _CMP_NEQ_UQ); // Note: _UQ here
    }
    boolvec_t operator<(const realvec& x) const
    {
      return _mm512_cmp_pd(v, x.v, _CMP_LT_OQ);
    }
    boolvec_t operator<=(const realvec& x) const
    {
      return _mm512_cmp_pd(v, x.v, _CMP_LE_OQ);
    }
    boolvec_t operator>(const realvec& x) const
    {
      return _mm512_cmp_pd(v, x.v, _CMP_GT_OQ);
    }
    boolvec_t operator>=(const realvec& x) const
    {
      return _mm512_cmp_pd(v, x.v, _CMP_GE_OQ);
    }
    
    
    
    realvec acos() const { return MF::vml_acos(*this); }
    realvec acosh() const { return MF::vml_acosh(*this); }
    realvec asin() const { return MF::vml_asin(*this); }
    realvec asinh() const { return MF::vml_asinh(*this); }
    realvec atan() const { return MF::vml_atan(*this); }
    realvec atan2(realvec y) const { return MF::vml_atan2(*this, y); }
    realvec atanh() const { return MF::vml_atanh(*this); }
    realvec cbrt() const { return MF::vml_cbrt(*this); }
    realvec ceil() const { return _mm512_ceil_pd(v); }
    realvec copysign(realvec y) const { return MF::vml_copysign(*this, y); }
    realvec cos() const { return MF::vml_cos(*this); }
    realvec cosh() const { return MF::vml_cosh(*this); }
    realvec exp() const { return MF::vml_exp(*this); }
    realvec exp10() const { return MF::vml_exp10(*this); }
    realvec exp2() const { return MF::vml_exp2(*this); }
    realvec expm1() const { return MF::vml_expm1(*this); }
    realvec fabs() const { return MF::vml_fabs(*this); }
    realvec fdim(realvec y) const { return MF::vml_fdim(*this, y); }
    realvec floor() const { return _mm512_floor_pd(v); }
    realvec fma(realvec y, realvec z) const
    {
      return _mm512_fmadd_pd(v, x.v, y.v);
    }
    realvec fmax(realvec y) const { return _mm512_gmax_pd(v, y.v); }
    realvec fmin(realvec y) const { return _mm512_gmin_pd(v, y.v); }
    realvec fmod(realvec y) const { return MF::vml_fmod(*this, y); }
    realvec frexp(intvec_t& r) const { return MF::vml_frexp(*this, r); }
    realvec hypot(realvec y) const { return MF::vml_hypot(*this, y); }
    intvec_t ilogb() const { return MF::vml_ilogb(*this); }
    boolvec_t isfinite() const { return MF::vml_isfinite(*this); }
    boolvec_t isinf() const { return MF::vml_isinf(*this); }
    boolvec_t isnan() const
    {
#ifdef VML_HAVE_NAN
      return _mm512_cmp_pd(v, v, _CMP_UNORD_Q);
#else
      return BV(false);
#endif
    }
    boolvec_t isnormal() const { return MF::vml_isnormal(*this); }
    realvec ldexp(int_t n) const { return MF::vml_ldexp(*this, n); }
    realvec ldexp(intvec_t n) const { return MF::vml_ldexp(*this, n); }
    realvec log() const { return MF::vml_log(*this); }
    realvec log10() const { return MF::vml_log10(*this); }
    realvec log1p() const { return MF::vml_log1p(*this); }
    realvec log2() const { return MF::vml_log2(*this); }
    realvec nextafter(realvec y) const { return MF::vml_nextafter(*this, y); }
    realvec pow(realvec y) const { return MF::vml_pow(*this, y); }
    realvec rcp() const { return _mm512_div_pd(_mm512_set1_pd(1.0), v); }
    realvec remainder(realvec y) const { return MF::vml_remainder(*this, y); }
    realvec rint() const
    {
      return _mm512_round_pd(v, _MM_FROUND_TO_NEAREST_INT);
    }
    realvec round() const { return MF::vml_round(*this); }
    realvec rsqrt() const { return MF::vml_rsqrt(*this); }
    boolvec_t signbit() const { return as_int().signbit(); }
    realvec sin() const { return MF::vml_sin(*this); }
    realvec sinh() const { return MF::vml_sinh(*this); }
    realvec sqrt() const { return _mm512_sqrt_pd(v); }
    realvec tan() const { return MF::vml_tan(*this); }
    realvec tanh() const { return MF::vml_tanh(*this); }
    realvec trunc() const { return _mm512_round_pd(v, _MM_FROUND_TO_ZERO); }
  };
  
  
  
  // boolvec definitions
  
  inline intvec<double,4> boolvec<double,4>::as_int() const
  {
    return _mm512_castpd_si512(v);
  }
  
  inline intvec<double,4> boolvec<double,4>::convert_int() const
  {
    return ifthen(v, IV(I(1)), IV(I(0)));
  }
  
  inline
  boolvec<double,4> boolvec<double,4>::ifthen(boolvec_t x, boolvec_t y) const
  {
    return (v & x.v) | (~v & y.v);
  }
  
  inline
  intvec<double,4> boolvec<double,4>::ifthen(intvec_t x, intvec_t y) const
  {
    return _mm512_blend_epi64(v, y.v, x.v)
  }
  
  inline
  realvec<double,4> boolvec<double,4>::ifthen(realvec_t x, realvec_t y) const
  {
    return _mm512_blend_pd(v, y.v, x.v)
  }
  
  
  
  // intvec definitions
  
  inline realvec<double,4> intvec<double,4>::as_float() const
  {
    return _mm512_castsi512_pd(v);
  }
  
  inline realvec<double,4> intvec<double,4>::convert_float() const
  {
    intvec_t r(_mm512_undefined_pd());
    for (int n=0; n<size; ++n) {
      r.set_elt(n, floatprops::convert_float((*this)[n]));
    }
    return r;
  }
  
} // namespace vecmathlib

#endif  // #ifndef VEC_MIC_DOUBLE8_H