/*
* Copyright (c) 2023, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <arm_neon.h>
#include "aom_dsp/arm/aom_neon_sve_bridge.h"
#include "aom_dsp/arm/mem_neon.h"
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
static inline uint64_t aom_sum_squares_2d_i16_4xh_sve(
const int16_t *src,
int stride,
int height) {
int64x2_t sum_squares = vdupq_n_s64(
0);
do {
int16x8_t s = vcombine_s16(vld1_s16(src), vld1_s16(src + stride));
sum_squares = aom_sdotq_s16(sum_squares, s, s);
src +=
2 * stride;
height -=
2;
}
while (height !=
0);
return (uint64_t)vaddvq_s64(sum_squares);
}
static inline uint64_t aom_sum_squares_2d_i16_8xh_sve(
const int16_t *src,
int stride,
int height) {
int64x2_t sum_squares[
2] = { vdupq_n_s64(
0), vdupq_n_s64(
0) };
do {
int16x8_t s0 = vld1q_s16(src +
0 * stride);
int16x8_t s1 = vld1q_s16(src +
1 * stride);
sum_squares[
0] = aom_sdotq_s16(sum_squares[
0], s0, s0);
sum_squares[
1] = aom_sdotq_s16(sum_squares[
1], s1, s1);
src +=
2 * stride;
height -=
2;
}
while (height !=
0);
sum_squares[
0] = vaddq_s64(sum_squares[
0], sum_squares[
1]);
return (uint64_t)vaddvq_s64(sum_squares[
0]);
}
static inline uint64_t aom_sum_squares_2d_i16_large_sve(
const int16_t *src,
int stride,
int width,
int height) {
int64x2_t sum_squares[
2] = { vdupq_n_s64(
0), vdupq_n_s64(
0) };
do {
const int16_t *src_ptr = src;
int w = width;
do {
int16x8_t s0 = vld1q_s16(src_ptr);
int16x8_t s1 = vld1q_s16(src_ptr +
8);
sum_squares[
0] = aom_sdotq_s16(sum_squares[
0], s0, s0);
sum_squares[
1] = aom_sdotq_s16(sum_squares[
1], s1, s1);
src_ptr +=
16;
w -=
16;
}
while (w !=
0);
src += stride;
}
while (--height !=
0);
sum_squares[
0] = vaddq_s64(sum_squares[
0], sum_squares[
1]);
return (uint64_t)vaddvq_s64(sum_squares[
0]);
}
static inline uint64_t aom_sum_squares_2d_i16_wxh_sve(
const int16_t *src,
int stride,
int width,
int height) {
svint64_t sum_squares = svdup_n_s64(
0);
uint64_t step = svcnth();
do {
const int16_t *src_ptr = src;
int w =
0;
do {
svbool_t pred = svwhilelt_b16_u32(w, width);
svint16_t s0 = svld1_s16(pred, src_ptr);
sum_squares = svdot_s64(sum_squares, s0, s0);
src_ptr += step;
w += step;
}
while (w < width);
src += stride;
}
while (--height !=
0);
return (uint64_t)svaddv_s64(svptrue_b64(), sum_squares);
}
uint64_t aom_sum_squares_2d_i16_sve(
const int16_t *src,
int stride,
int width,
int height) {
if (width ==
4) {
return aom_sum_squares_2d_i16_4xh_sve(src, stride, height);
}
if (width ==
8) {
return aom_sum_squares_2d_i16_8xh_sve(src, stride, height);
}
if (width %
16 ==
0) {
return aom_sum_squares_2d_i16_large_sve(src, stride, width, height);
}
return aom_sum_squares_2d_i16_wxh_sve(src, stride, width, height);
}
uint64_t aom_sum_squares_i16_sve(
const int16_t *src, uint32_t n) {
// This function seems to be called only for values of N >= 64. See
// av1/encoder/compound_type.c. Additionally, because N = width x height for
// width and height between the standard block sizes, N will also be a
// multiple of 64.
if (LIKELY(n %
64 ==
0)) {
int64x2_t sum[
4] = { vdupq_n_s64(
0), vdupq_n_s64(
0), vdupq_n_s64(
0),
vdupq_n_s64(
0) };
do {
int16x8_t s0 = vld1q_s16(src);
int16x8_t s1 = vld1q_s16(src +
8);
int16x8_t s2 = vld1q_s16(src +
16);
int16x8_t s3 = vld1q_s16(src +
24);
sum[
0] = aom_sdotq_s16(sum[
0], s0, s0);
sum[
1] = aom_sdotq_s16(sum[
1], s1, s1);
sum[
2] = aom_sdotq_s16(sum[
2], s2, s2);
sum[
3] = aom_sdotq_s16(sum[
3], s3, s3);
src +=
32;
n -=
32;
}
while (n !=
0);
sum[
0] = vaddq_s64(sum[
0], sum[
1]);
sum[
2] = vaddq_s64(sum[
2], sum[
3]);
sum[
0] = vaddq_s64(sum[
0], sum[
2]);
return vaddvq_s64(sum[
0]);
}
return aom_sum_squares_i16_c(src, n);
}
static inline uint64_t aom_sum_sse_2d_i16_4xh_sve(
const int16_t *src,
int stride,
int height,
int *sum) {
int64x2_t sse = vdupq_n_s64(
0);
int32x4_t sum_s32 = vdupq_n_s32(
0);
do {
int16x8_t s = vcombine_s16(vld1_s16(src), vld1_s16(src + stride));
sse = aom_sdotq_s16(sse, s, s);
sum_s32 = vpadalq_s16(sum_s32, s);
src +=
2 * stride;
height -=
2;
}
while (height !=
0);
*sum += vaddvq_s32(sum_s32);
return vaddvq_s64(sse);
}
static inline uint64_t aom_sum_sse_2d_i16_8xh_sve(
const int16_t *src,
int stride,
int height,
int *sum) {
int64x2_t sse[
2] = { vdupq_n_s64(
0), vdupq_n_s64(
0) };
int32x4_t sum_acc[
2] = { vdupq_n_s32(
0), vdupq_n_s32(
0) };
do {
int16x8_t s0 = vld1q_s16(src);
int16x8_t s1 = vld1q_s16(src + stride);
sse[
0] = aom_sdotq_s16(sse[
0], s0, s0);
sse[
1] = aom_sdotq_s16(sse[
1], s1, s1);
sum_acc[
0] = vpadalq_s16(sum_acc[
0], s0);
sum_acc[
1] = vpadalq_s16(sum_acc[
1], s1);
src +=
2 * stride;
height -=
2;
}
while (height !=
0);
*sum += vaddvq_s32(vaddq_s32(sum_acc[
0], sum_acc[
1]));
return vaddvq_s64(vaddq_s64(sse[
0], sse[
1]));
}
static inline uint64_t aom_sum_sse_2d_i16_16xh_sve(
const int16_t *src,
int stride,
int width,
int height,
int *sum) {
int64x2_t sse[
2] = { vdupq_n_s64(
0), vdupq_n_s64(
0) };
int32x4_t sum_acc[
2] = { vdupq_n_s32(
0), vdupq_n_s32(
0) };
do {
int w =
0;
do {
int16x8_t s0 = vld1q_s16(src + w);
int16x8_t s1 = vld1q_s16(src + w +
8);
sse[
0] = aom_sdotq_s16(sse[
0], s0, s0);
sse[
1] = aom_sdotq_s16(sse[
1], s1, s1);
sum_acc[
0] = vpadalq_s16(sum_acc[
0], s0);
sum_acc[
1] = vpadalq_s16(sum_acc[
1], s1);
w +=
16;
}
while (w < width);
src += stride;
}
while (--height !=
0);
*sum += vaddvq_s32(vaddq_s32(sum_acc[
0], sum_acc[
1]));
return vaddvq_s64(vaddq_s64(sse[
0], sse[
1]));
}
uint64_t aom_sum_sse_2d_i16_sve(
const int16_t *src,
int stride,
int width,
int height,
int *sum) {
uint64_t sse;
if (width ==
4) {
sse = aom_sum_sse_2d_i16_4xh_sve(src, stride, height, sum);
}
else if (width ==
8) {
sse = aom_sum_sse_2d_i16_8xh_sve(src, stride, height, sum);
}
else if (width %
16 ==
0) {
sse = aom_sum_sse_2d_i16_16xh_sve(src, stride, width, height, sum);
}
else {
sse = aom_sum_sse_2d_i16_c(src, stride, width, height, sum);
}
return sse;
}
#if CONFIG_AV1_HIGHBITDEPTH
static inline uint64_t aom_var_2d_u16_4xh_sve(uint8_t *src,
int src_stride,
int width,
int height) {
uint16_t *src_u16 = CONVERT_TO_SHORTPTR(src);
uint64_t sum =
0;
uint64_t sse =
0;
uint32x4_t sum_u32 = vdupq_n_u32(
0);
uint64x2_t sse_u64 = vdupq_n_u64(
0);
int h = height;
do {
uint16x8_t s0 =
vcombine_u16(vld1_u16(src_u16), vld1_u16(src_u16 + src_stride));
sum_u32 = vpadalq_u16(sum_u32, s0);
sse_u64 = aom_udotq_u16(sse_u64, s0, s0);
src_u16 +=
2 * src_stride;
h -=
2;
}
while (h !=
0);
sum += vaddlvq_u32(sum_u32);
sse += vaddvq_u64(sse_u64);
return sse - sum * sum / (width * height);
}
static inline uint64_t aom_var_2d_u16_8xh_sve(uint8_t *src,
int src_stride,
int width,
int height) {
uint16_t *src_u16 = CONVERT_TO_SHORTPTR(src);
uint64_t sum =
0;
uint64_t sse =
0;
uint32x4_t sum_u32 = vdupq_n_u32(
0);
uint64x2_t sse_u64 = vdupq_n_u64(
0);
int h = height;
do {
int w = width;
uint16_t *src_ptr = src_u16;
do {
uint16x8_t s0 = vld1q_u16(src_ptr);
sum_u32 = vpadalq_u16(sum_u32, s0);
sse_u64 = aom_udotq_u16(sse_u64, s0, s0);
src_ptr +=
8;
w -=
8;
}
while (w !=
0);
src_u16 += src_stride;
}
while (--h !=
0);
sum += vaddlvq_u32(sum_u32);
sse += vaddvq_u64(sse_u64);
return sse - sum * sum / (width * height);
}
static inline uint64_t aom_var_2d_u16_16xh_sve(uint8_t *src,
int src_stride,
int width,
int height) {
uint16_t *src_u16 = CONVERT_TO_SHORTPTR(src);
uint64_t sum =
0;
uint64_t sse =
0;
uint32x4_t sum_u32[
2] = { vdupq_n_u32(
0), vdupq_n_u32(
0) };
uint64x2_t sse_u64[
2] = { vdupq_n_u64(
0), vdupq_n_u64(
0) };
int h = height;
do {
int w = width;
uint16_t *src_ptr = src_u16;
do {
uint16x8_t s0 = vld1q_u16(src_ptr);
uint16x8_t s1 = vld1q_u16(src_ptr +
8);
sum_u32[
0] = vpadalq_u16(sum_u32[
0], s0);
sum_u32[
1] = vpadalq_u16(sum_u32[
1], s1);
sse_u64[
0] = aom_udotq_u16(sse_u64[
0], s0, s0);
sse_u64[
1] = aom_udotq_u16(sse_u64[
1], s1, s1);
src_ptr +=
16;
w -=
16;
}
while (w !=
0);
src_u16 += src_stride;
}
while (--h !=
0);
sum_u32[
0] = vaddq_u32(sum_u32[
0], sum_u32[
1]);
sse_u64[
0] = vaddq_u64(sse_u64[
0], sse_u64[
1]);
sum += vaddlvq_u32(sum_u32[
0]);
sse += vaddvq_u64(sse_u64[
0]);
return sse - sum * sum / (width * height);
}
static inline uint64_t aom_var_2d_u16_large_sve(uint8_t *src,
int src_stride,
int width,
int height) {
uint16_t *src_u16 = CONVERT_TO_SHORTPTR(src);
uint64_t sum =
0;
uint64_t sse =
0;
uint32x4_t sum_u32[
4] = { vdupq_n_u32(
0), vdupq_n_u32(
0), vdupq_n_u32(
0),
vdupq_n_u32(
0) };
uint64x2_t sse_u64[
4] = { vdupq_n_u64(
0), vdupq_n_u64(
0), vdupq_n_u64(
0),
vdupq_n_u64(
0) };
int h = height;
do {
int w = width;
uint16_t *src_ptr = src_u16;
do {
uint16x8_t s0 = vld1q_u16(src_ptr);
uint16x8_t s1 = vld1q_u16(src_ptr +
8);
uint16x8_t s2 = vld1q_u16(src_ptr +
16);
uint16x8_t s3 = vld1q_u16(src_ptr +
24);
sum_u32[
0] = vpadalq_u16(sum_u32[
0], s0);
sum_u32[
1] = vpadalq_u16(sum_u32[
1], s1);
sum_u32[
2] = vpadalq_u16(sum_u32[
2], s2);
sum_u32[
3] = vpadalq_u16(sum_u32[
3], s3);
sse_u64[
0] = aom_udotq_u16(sse_u64[
0], s0, s0);
sse_u64[
1] = aom_udotq_u16(sse_u64[
1], s1, s1);
sse_u64[
2] = aom_udotq_u16(sse_u64[
2], s2, s2);
sse_u64[
3] = aom_udotq_u16(sse_u64[
3], s3, s3);
src_ptr +=
32;
w -=
32;
}
while (w !=
0);
src_u16 += src_stride;
}
while (--h !=
0);
sum_u32[
0] = vaddq_u32(sum_u32[
0], sum_u32[
1]);
sum_u32[
2] = vaddq_u32(sum_u32[
2], sum_u32[
3]);
sum_u32[
0] = vaddq_u32(sum_u32[
0], sum_u32[
2]);
sse_u64[
0] = vaddq_u64(sse_u64[
0], sse_u64[
1]);
sse_u64[
2] = vaddq_u64(sse_u64[
2], sse_u64[
3]);
sse_u64[
0] = vaddq_u64(sse_u64[
0], sse_u64[
2]);
sum += vaddlvq_u32(sum_u32[
0]);
sse += vaddvq_u64(sse_u64[
0]);
return sse - sum * sum / (width * height);
}
uint64_t aom_var_2d_u16_sve(uint8_t *src,
int src_stride,
int width,
int height) {
if (width ==
4) {
return aom_var_2d_u16_4xh_sve(src, src_stride, width, height);
}
if (width ==
8) {
return aom_var_2d_u16_8xh_sve(src, src_stride, width, height);
}
if (width ==
16) {
return aom_var_2d_u16_16xh_sve(src, src_stride, width, height);
}
if (width %
32 ==
0) {
return aom_var_2d_u16_large_sve(src, src_stride, width, height);
}
return aom_var_2d_u16_neon(src, src_stride, width, height);
}
#endif // CONFIG_AV1_HIGHBITDEPTH
