volk_32fc_accumulator_s32fc.h

Go to the documentation of this file.

/* -*- c++ -*- */
/*
 * Copyright 2019 Free Software Foundation, Inc.
 *
 * This file is part of VOLK
 *
 * SPDX-License-Identifier: LGPL-3.0-or-later
 */
 
#ifndef INCLUDED_volk_32fc_accumulator_s32fc_a_H
#define INCLUDED_volk_32fc_accumulator_s32fc_a_H
 
#include <inttypes.h>
#include <volk/volk_common.h>
 
#ifdef LV_HAVE_AVX512F
#include <immintrin.h>
 
static inline void volk_32fc_accumulator_s32fc_a_avx512f(lv_32fc_t* result,
                                                         const lv_32fc_t* inputBuffer,
                                                         unsigned int num_points)
{
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int number = 0;
    const unsigned int eighthPoints = num_points / 8;
 
    const lv_32fc_t* aPtr = inputBuffer;
    __VOLK_ATTR_ALIGNED(64) float tempBuffer[16];
 
    __m512 accumulator = _mm512_setzero_ps();
    __m512 aVal = _mm512_setzero_ps();
 
    for (; number < eighthPoints; number++) {
        aVal = _mm512_load_ps((float*)aPtr);
        accumulator = _mm512_add_ps(accumulator, aVal);
        aPtr += 8;
    }
 
    _mm512_store_ps(tempBuffer, accumulator);
 
    // Sum pairs as complex numbers
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
    returnValue += lv_cmake(tempBuffer[4], tempBuffer[5]);
    returnValue += lv_cmake(tempBuffer[6], tempBuffer[7]);
    returnValue += lv_cmake(tempBuffer[8], tempBuffer[9]);
    returnValue += lv_cmake(tempBuffer[10], tempBuffer[11]);
    returnValue += lv_cmake(tempBuffer[12], tempBuffer[13]);
    returnValue += lv_cmake(tempBuffer[14], tempBuffer[15]);
 
    number = eighthPoints * 8;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_AVX512F */
 
 
#ifdef LV_HAVE_AVX512F
#include <immintrin.h>
 
static inline void volk_32fc_accumulator_s32fc_u_avx512f(lv_32fc_t* result,
                                                         const lv_32fc_t* inputBuffer,
                                                         unsigned int num_points)
{
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int number = 0;
    const unsigned int eighthPoints = num_points / 8;
 
    const lv_32fc_t* aPtr = inputBuffer;
    __VOLK_ATTR_ALIGNED(64) float tempBuffer[16];
 
    __m512 accumulator = _mm512_setzero_ps();
    __m512 aVal = _mm512_setzero_ps();
 
    for (; number < eighthPoints; number++) {
        aVal = _mm512_loadu_ps((float*)aPtr);
        accumulator = _mm512_add_ps(accumulator, aVal);
        aPtr += 8;
    }
 
    _mm512_store_ps(tempBuffer, accumulator);
 
    // Sum pairs as complex numbers
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
    returnValue += lv_cmake(tempBuffer[4], tempBuffer[5]);
    returnValue += lv_cmake(tempBuffer[6], tempBuffer[7]);
    returnValue += lv_cmake(tempBuffer[8], tempBuffer[9]);
    returnValue += lv_cmake(tempBuffer[10], tempBuffer[11]);
    returnValue += lv_cmake(tempBuffer[12], tempBuffer[13]);
    returnValue += lv_cmake(tempBuffer[14], tempBuffer[15]);
 
    number = eighthPoints * 8;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_AVX512F */
 
 
#ifdef LV_HAVE_GENERIC
static inline void volk_32fc_accumulator_s32fc_generic(lv_32fc_t* result,
                                                       const lv_32fc_t* inputBuffer,
                                                       unsigned int num_points)
{
    const lv_32fc_t* aPtr = inputBuffer;
    unsigned int number = 0;
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
 
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_GENERIC */
 
#ifdef LV_HAVE_AVX
#include <immintrin.h>
 
static inline void volk_32fc_accumulator_s32fc_u_avx(lv_32fc_t* result,
                                                     const lv_32fc_t* inputBuffer,
                                                     unsigned int num_points)
{
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int number = 0;
    const unsigned int quarterPoints = num_points / 4;
 
    const lv_32fc_t* aPtr = inputBuffer;
    __VOLK_ATTR_ALIGNED(32) float tempBuffer[8];
 
    __m256 accumulator = _mm256_setzero_ps();
    __m256 aVal = _mm256_setzero_ps();
 
    for (; number < quarterPoints; number++) {
        aVal = _mm256_loadu_ps((float*)aPtr);
        accumulator = _mm256_add_ps(accumulator, aVal);
        aPtr += 4;
    }
 
    _mm256_store_ps(tempBuffer, accumulator);
 
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
    returnValue += lv_cmake(tempBuffer[4], tempBuffer[5]);
    returnValue += lv_cmake(tempBuffer[6], tempBuffer[7]);
 
    number = quarterPoints * 4;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_AVX */
 
#ifdef LV_HAVE_SSE
#include <xmmintrin.h>
 
static inline void volk_32fc_accumulator_s32fc_u_sse(lv_32fc_t* result,
                                                     const lv_32fc_t* inputBuffer,
                                                     unsigned int num_points)
{
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int number = 0;
    const unsigned int halfPoints = num_points / 2;
 
    const lv_32fc_t* aPtr = inputBuffer;
    __VOLK_ATTR_ALIGNED(16) float tempBuffer[4];
 
    __m128 accumulator = _mm_setzero_ps();
    __m128 aVal = _mm_setzero_ps();
 
    for (; number < halfPoints; number++) {
        aVal = _mm_loadu_ps((float*)aPtr);
        accumulator = _mm_add_ps(accumulator, aVal);
        aPtr += 2;
    }
 
    _mm_store_ps(tempBuffer, accumulator);
 
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
 
    number = halfPoints * 2;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_SSE */
 
#ifdef LV_HAVE_AVX
#include <immintrin.h>
 
static inline void volk_32fc_accumulator_s32fc_a_avx(lv_32fc_t* result,
                                                     const lv_32fc_t* inputBuffer,
                                                     unsigned int num_points)
{
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int number = 0;
    const unsigned int quarterPoints = num_points / 4;
 
    const lv_32fc_t* aPtr = inputBuffer;
    __VOLK_ATTR_ALIGNED(32) float tempBuffer[8];
 
    __m256 accumulator = _mm256_setzero_ps();
    __m256 aVal = _mm256_setzero_ps();
 
    for (; number < quarterPoints; number++) {
        aVal = _mm256_load_ps((float*)aPtr);
        accumulator = _mm256_add_ps(accumulator, aVal);
        aPtr += 4;
    }
 
    _mm256_store_ps(tempBuffer, accumulator);
 
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
    returnValue += lv_cmake(tempBuffer[4], tempBuffer[5]);
    returnValue += lv_cmake(tempBuffer[6], tempBuffer[7]);
 
    number = quarterPoints * 4;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_AVX */
 
#ifdef LV_HAVE_SSE
#include <xmmintrin.h>
 
static inline void volk_32fc_accumulator_s32fc_a_sse(lv_32fc_t* result,
                                                     const lv_32fc_t* inputBuffer,
                                                     unsigned int num_points)
{
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int number = 0;
    const unsigned int halfPoints = num_points / 2;
 
    const lv_32fc_t* aPtr = inputBuffer;
    __VOLK_ATTR_ALIGNED(16) float tempBuffer[4];
 
    __m128 accumulator = _mm_setzero_ps();
    __m128 aVal = _mm_setzero_ps();
 
    for (; number < halfPoints; number++) {
        aVal = _mm_load_ps((float*)aPtr);
        accumulator = _mm_add_ps(accumulator, aVal);
        aPtr += 2;
    }
 
    _mm_store_ps(tempBuffer, accumulator);
 
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
 
    number = halfPoints * 2;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_SSE */
 
#ifdef LV_HAVE_NEON
#include <arm_neon.h>
static inline void volk_32fc_accumulator_s32fc_neon(lv_32fc_t* result,
                                                    const lv_32fc_t* inputBuffer,
                                                    unsigned int num_points)
{
    const lv_32fc_t* aPtr = inputBuffer;
    unsigned int number = 0;
    lv_32fc_t returnValue = lv_cmake(0.f, 0.f);
    unsigned int eighthPoints = num_points / 8;
    float32x4_t in_vec;
    float32x4_t out_vec0 = { 0.f, 0.f, 0.f, 0.f };
    float32x4_t out_vec1 = { 0.f, 0.f, 0.f, 0.f };
    float32x4_t out_vec2 = { 0.f, 0.f, 0.f, 0.f };
    float32x4_t out_vec3 = { 0.f, 0.f, 0.f, 0.f };
    __VOLK_ATTR_ALIGNED(32) float tempBuffer[4];
 
    for (; number < eighthPoints; number++) {
        in_vec = vld1q_f32((float*)aPtr);
        out_vec0 = vaddq_f32(in_vec, out_vec0);
        aPtr += 2;
 
        in_vec = vld1q_f32((float*)aPtr);
        out_vec1 = vaddq_f32(in_vec, out_vec1);
        aPtr += 2;
 
        in_vec = vld1q_f32((float*)aPtr);
        out_vec2 = vaddq_f32(in_vec, out_vec2);
        aPtr += 2;
 
        in_vec = vld1q_f32((float*)aPtr);
        out_vec3 = vaddq_f32(in_vec, out_vec3);
        aPtr += 2;
    }
    vst1q_f32(tempBuffer, out_vec0);
    returnValue = lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
 
    vst1q_f32(tempBuffer, out_vec1);
    returnValue += lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
 
    vst1q_f32(tempBuffer, out_vec2);
    returnValue += lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
 
    vst1q_f32(tempBuffer, out_vec3);
    returnValue += lv_cmake(tempBuffer[0], tempBuffer[1]);
    returnValue += lv_cmake(tempBuffer[2], tempBuffer[3]);
 
    number = eighthPoints * 8;
    for (; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
    *result = returnValue;
}
#endif /* LV_HAVE_NEON */
 
#ifdef LV_HAVE_NEONV8
#include <arm_neon.h>
 
static inline void volk_32fc_accumulator_s32fc_neonv8(lv_32fc_t* result,
                                                      const lv_32fc_t* inputBuffer,
                                                      unsigned int num_points)
{
    const lv_32fc_t* aPtr = inputBuffer;
    unsigned int number = 0;
    const unsigned int eighthPoints = num_points / 8;
 
    /* Keep interleaved like neon version - vld1q is faster than vld2q */
    float32x4_t in_vec;
    float32x4_t out_vec0 = vdupq_n_f32(0.f);
    float32x4_t out_vec1 = vdupq_n_f32(0.f);
    float32x4_t out_vec2 = vdupq_n_f32(0.f);
    float32x4_t out_vec3 = vdupq_n_f32(0.f);
 
    for (; number < eighthPoints; number++) {
        in_vec = vld1q_f32((float*)aPtr);
        out_vec0 = vaddq_f32(in_vec, out_vec0);
        aPtr += 2;
 
        in_vec = vld1q_f32((float*)aPtr);
        out_vec1 = vaddq_f32(in_vec, out_vec1);
        aPtr += 2;
 
        in_vec = vld1q_f32((float*)aPtr);
        out_vec2 = vaddq_f32(in_vec, out_vec2);
        aPtr += 2;
 
        in_vec = vld1q_f32((float*)aPtr);
        out_vec3 = vaddq_f32(in_vec, out_vec3);
        aPtr += 2;
    }
 
    /* Combine the 4 accumulators */
    out_vec0 = vaddq_f32(out_vec0, out_vec1);
    out_vec2 = vaddq_f32(out_vec2, out_vec3);
    out_vec0 = vaddq_f32(out_vec0, out_vec2);
 
    /* Horizontal reduction: out_vec0 = [sum_r0, sum_i0, sum_r1, sum_i1] */
    /* We need real = sum_r0 + sum_r1, imag = sum_i0 + sum_i1 */
    float32x2_t low = vget_low_f32(out_vec0);   /* [sum_r0, sum_i0] */
    float32x2_t high = vget_high_f32(out_vec0); /* [sum_r1, sum_i1] */
    float32x2_t sum = vadd_f32(low, high);      /* [real_sum, imag_sum] */
 
    lv_32fc_t returnValue = lv_cmake(vget_lane_f32(sum, 0), vget_lane_f32(sum, 1));
 
    /* Tail case */
    for (number = eighthPoints * 8; number < num_points; number++) {
        returnValue += (*aPtr++);
    }
 
    *result = returnValue;
}
 
#endif /* LV_HAVE_NEONV8 */
 
#ifdef LV_HAVE_RVV
#include <riscv_vector.h>
#include <volk/volk_rvv_intrinsics.h>
 
static inline void volk_32fc_accumulator_s32fc_rvv(lv_32fc_t* result,
                                                   const lv_32fc_t* inputBuffer,
                                                   unsigned int num_points)
{
    size_t vlmax = __riscv_vsetvlmax_e32m8();
    vfloat32m8_t vsum = __riscv_vfmv_v_f_f32m8(0, vlmax);
    const float* in = (const float*)inputBuffer;
    size_t n = num_points * 2;
    for (size_t vl; n > 0; n -= vl, in += vl) {
        vl = __riscv_vsetvl_e32m8(n < vlmax ? n : vlmax); /* force exact vl */
        vfloat32m8_t v = __riscv_vle32_v_f32m8(in, vl);
        vsum = __riscv_vfadd_tu(vsum, vsum, v, vl);
    }
    vuint64m8_t vsumu = __riscv_vreinterpret_u64m8(__riscv_vreinterpret_u32m8(vsum));
    vfloat32m4_t vsum1 = __riscv_vreinterpret_f32m4(__riscv_vnsrl(vsumu, 0, vlmax));
    vfloat32m4_t vsum2 = __riscv_vreinterpret_f32m4(__riscv_vnsrl(vsumu, 32, vlmax));
    vlmax = __riscv_vsetvlmax_e32m1();
    vfloat32m1_t vr = RISCV_SHRINK4(vfadd, f, 32, vsum1);
    vfloat32m1_t vi = RISCV_SHRINK4(vfadd, f, 32, vsum2);
    vfloat32m1_t z = __riscv_vfmv_s_f_f32m1(0, vlmax);
    *result = lv_cmake(__riscv_vfmv_f(__riscv_vfredusum(vr, z, vlmax)),
                       __riscv_vfmv_f(__riscv_vfredusum(vi, z, vlmax)));
}
#endif /*LV_HAVE_RVV*/
 
#endif /* INCLUDED_volk_32fc_accumulator_s32fc_a_H */