/*
* Copyright (c) 2023, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <arm_neon.h>
#include "config/aom_dsp_rtcd.h"
#include "aom_dsp/arm/sum_neon.h"
static inline void highbd_sse_8x1_init_neon(
const uint16_t *src,
const uint16_t *ref,
uint32x4_t *sse_acc0,
uint32x4_t *sse_acc1) {
uint16x8_t s = vld1q_u16(src);
uint16x8_t r = vld1q_u16(ref);
uint16x8_t abs_diff = vabdq_u16(s, r);
uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
*sse_acc0 = vmull_u16(abs_diff_lo, abs_diff_lo);
*sse_acc1 = vmull_u16(abs_diff_hi, abs_diff_hi);
}
static inline void highbd_sse_8x1_neon(
const uint16_t *src,
const uint16_t *ref,
uint32x4_t *sse_acc0,
uint32x4_t *sse_acc1) {
uint16x8_t s = vld1q_u16(src);
uint16x8_t r = vld1q_u16(ref);
uint16x8_t abs_diff = vabdq_u16(s, r);
uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
*sse_acc0 = vmlal_u16(*sse_acc0, abs_diff_lo, abs_diff_lo);
*sse_acc1 = vmlal_u16(*sse_acc1, abs_diff_hi, abs_diff_hi);
}
static inline int64_t highbd_sse_128xh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int height) {
uint32x4_t sse[
16];
highbd_sse_8x1_init_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
='color: green'>1]);
highbd_sse_8x1_init_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
='color: green'>3]);
highbd_sse_8x1_init_neon(src +
2 *
8, ref +
2 *
8, &sse[
4], &sse[
='color: green'>5]);
highbd_sse_8x1_init_neon(src +
3 *
8, ref +
3 *
8, &sse[
6], &sse[
='color: green'>7]);
highbd_sse_8x1_init_neon(src +
4 *
8, ref +
4 *
8, &sse[
8], &sse[
='color: green'>9]);
highbd_sse_8x1_init_neon(src +
5 *
8, ref +
5 *
8, &sse[
10], &sse[
e='color: green'>11]);
highbd_sse_8x1_init_neon(src +
6 *
8, ref +
6 *
8, &sse[
12], &sse[
e='color: green'>13]);
highbd_sse_8x1_init_neon(src +
7 *
8, ref +
7 *
8, &sse[
14], &sse[
e='color: green'>15]);
highbd_sse_8x1_neon(src +
8 *
8, ref +
8 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
9 *
8, ref +
9 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
10 *
8, ref +
10 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
11 *
8, ref +
11 *
8, &sse[
6], &sse[
7]);
highbd_sse_8x1_neon(src +
12 *
8, ref +
12 *
8, &sse[
8], &sse[
9]);
highbd_sse_8x1_neon(src +
13 *
8, ref +
13 *
8, &sse[
10], &sse[
11]);
highbd_sse_8x1_neon(src +
14 *
8, ref +
14 *
8, &sse[
12], &sse[
13]);
highbd_sse_8x1_neon(src +
15 *
8, ref +
15 *
8, &sse[
14], &sse[
15]);
src += src_stride;
ref += ref_stride;
while (--height !=
0) {
highbd_sse_8x1_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
2 *
8, ref +
2 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
3 *
8, ref +
3 *
8, &sse[
6], &sse[
7]);
highbd_sse_8x1_neon(src +
4 *
8, ref +
4 *
8, &sse[
8], &sse[
9]);
highbd_sse_8x1_neon(src +
5 *
8, ref +
5 *
8, &sse[
10], &sse[
11]);
highbd_sse_8x1_neon(src +
6 *
8, ref +
6 *
8, &sse[
12], &sse[
13]);
highbd_sse_8x1_neon(src +
7 *
8, ref +
7 *
8, &sse[
14], &sse[
15]);
highbd_sse_8x1_neon(src +
8 *
8, ref +
8 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
9 *
8, ref +
9 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
10 *
8, ref +
10 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
11 *
8, ref +
11 *
8, &sse[
6], &sse[
7]);
highbd_sse_8x1_neon(src +
12 *
8, ref +
12 *
8, &sse[
8], &sse[
9]);
highbd_sse_8x1_neon(src +
13 *
8, ref +
13 *
8, &sse[
10], &sse[
11]);
highbd_sse_8x1_neon(src +
14 *
8, ref +
14 *
8, &sse[
12], &sse[
13]);
highbd_sse_8x1_neon(src +
15 *
8, ref +
15 *
8, &sse[
14], &sse[
15]);
src += src_stride;
ref += ref_stride;
}
return horizontal_long_add_u32x4_x16(sse);
}
static inline int64_t highbd_sse_64xh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int height) {
uint32x4_t sse[
8];
highbd_sse_8x1_init_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
='color: green'>1]);
highbd_sse_8x1_init_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
='color: green'>3]);
highbd_sse_8x1_init_neon(src +
2 *
8, ref +
2 *
8, &sse[
4], &sse[
='color: green'>5]);
highbd_sse_8x1_init_neon(src +
3 *
8, ref +
3 *
8, &sse[
6], &sse[
='color: green'>7]);
highbd_sse_8x1_neon(src +
4 *
8, ref +
4 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
5 *
8, ref +
5 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
6 *
8, ref +
6 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
7 *
8, ref +
7 *
8, &sse[
6], &sse[
7]);
src += src_stride;
ref += ref_stride;
while (--height !=
0) {
highbd_sse_8x1_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
2 *
8, ref +
2 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
3 *
8, ref +
3 *
8, &sse[
6], &sse[
7]);
highbd_sse_8x1_neon(src +
4 *
8, ref +
4 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
5 *
8, ref +
5 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
6 *
8, ref +
6 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
7 *
8, ref +
7 *
8, &sse[
6], &sse[
7]);
src += src_stride;
ref += ref_stride;
}
return horizontal_long_add_u32x4_x8(sse);
}
static inline int64_t highbd_sse_32xh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int height) {
uint32x4_t sse[
8];
highbd_sse_8x1_init_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
='color: green'>1]);
highbd_sse_8x1_init_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
='color: green'>3]);
highbd_sse_8x1_init_neon(src +
2 *
8, ref +
2 *
8, &sse[
4], &sse[
='color: green'>5]);
highbd_sse_8x1_init_neon(src +
3 *
8, ref +
3 *
8, &sse[
6], &sse[
='color: green'>7]);
src += src_stride;
ref += ref_stride;
while (--height !=
0) {
highbd_sse_8x1_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
3]);
highbd_sse_8x1_neon(src +
2 *
8, ref +
2 *
8, &sse[
4], &sse[
5]);
highbd_sse_8x1_neon(src +
3 *
8, ref +
3 *
8, &sse[
6], &sse[
7]);
src += src_stride;
ref += ref_stride;
}
return horizontal_long_add_u32x4_x8(sse);
}
static inline int64_t highbd_sse_16xh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int height) {
uint32x4_t sse[
4];
highbd_sse_8x1_init_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
='color: green'>1]);
highbd_sse_8x1_init_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
='color: green'>3]);
src += src_stride;
ref += ref_stride;
while (--height !=
0) {
highbd_sse_8x1_neon(src +
0 *
8, ref +
0 *
8, &sse[
0], &sse[
1]);
highbd_sse_8x1_neon(src +
1 *
8, ref +
1 *
8, &sse[
2], &sse[
3]);
src += src_stride;
ref += ref_stride;
}
return horizontal_long_add_u32x4_x4(sse);
}
static inline int64_t highbd_sse_8xh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int height) {
uint32x4_t sse[
2];
highbd_sse_8x1_init_neon(src, ref, &sse[
0], &sse[
1]);
src += src_stride;
ref += ref_stride;
while (--height !=
0) {
highbd_sse_8x1_neon(src, ref, &sse[
0], &sse[
1]);
src += src_stride;
ref += ref_stride;
}
return horizontal_long_add_u32x4_x2(sse);
}
static inline int64_t highbd_sse_4xh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int height) {
// Peel the first loop iteration.
uint16x4_t s = vld1_u16(src);
uint16x4_t r = vld1_u16(ref);
uint16x4_t abs_diff = vabd_u16(s, r);
uint32x4_t sse = vmull_u16(abs_diff, abs_diff);
src += src_stride;
ref += ref_stride;
while (--height !=
0) {
s = vld1_u16(src);
r = vld1_u16(ref);
abs_diff = vabd_u16(s, r);
sse = vmlal_u16(sse, abs_diff, abs_diff);
src += src_stride;
ref += ref_stride;
}
return horizontal_long_add_u32x4(sse);
}
static inline int64_t highbd_sse_wxh_neon(
const uint16_t *src,
int src_stride,
const uint16_t *ref,
int ref_stride,
int width,
int height) {
// { 0, 1, 2, 3, 4, 5, 6, 7 }
uint16x8_t k01234567 = vmovl_u8(vcreate_u8(
0x0706050403020100));
uint16x8_t remainder_mask = vcltq_u16(k01234567, vdupq_n_u16(width &
7));
uint64_t sse =
0;
do {
int w = width;
int offset =
0;
do {
uint16x8_t s = vld1q_u16(src + offset);
uint16x8_t r = vld1q_u16(ref + offset);
if (w <
8) {
// Mask out-of-range elements.
s = vandq_u16(s, remainder_mask);
r = vandq_u16(r, remainder_mask);
}
uint16x8_t abs_diff = vabdq_u16(s, r);
uint16x4_t abs_diff_lo = vget_low_u16(abs_diff);
uint16x4_t abs_diff_hi = vget_high_u16(abs_diff);
uint32x4_t sse_u32 = vmull_u16(abs_diff_lo, abs_diff_lo);
sse_u32 = vmlal_u16(sse_u32, abs_diff_hi, abs_diff_hi);
sse += horizontal_long_add_u32x4(sse_u32);
offset +=
8;
w -=
8;
}
while (w >
0);
src += src_stride;
ref += ref_stride;
}
while (--height !=
0);
return sse;
}
int64_t aom_highbd_sse_neon(
const uint8_t *src8,
int src_stride,
const uint8_t *ref8,
int ref_stride,
int width,
int height) {
uint16_t *src = CONVERT_TO_SHORTPTR(src8);
uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
switch (width) {
case 4:
return highbd_sse_4xh_neon(src, src_stride, ref, ref_stride, height);
case 8:
return highbd_sse_8xh_neon(src, src_stride, ref, ref_stride, height);
case 16:
return highbd_sse_16xh_neon(src, src_stride, ref, ref_stride, height);
case 32:
return highbd_sse_32xh_neon(src, src_stride, ref, ref_stride, height);
case 64:
return highbd_sse_64xh_neon(src, src_stride, ref, ref_stride, height);
case 128:
return highbd_sse_128xh_neon(src, src_stride, ref, ref_stride, height);
default:
return highbd_sse_wxh_neon(src, src_stride, ref, ref_stride, width,
height);
}
}
