上传完整项目文件

This commit is contained in:
xhgit committed 2026-06-21 14:26:15 +08:00
1 parent e63d58df8d
commit aa0dd45839
8663 files changed
+2393125

No files matched your search

+76
View File
@@ -0,0 +1,76 @@
# subsystems
OBJS-$(CONFIG_AC3DSP) += aarch64/ac3dsp_init_aarch64.o
OBJS-$(CONFIG_FDCTDSP) += aarch64/fdctdsp_init_aarch64.o
OBJS-$(CONFIG_FMTCONVERT) += aarch64/fmtconvert_init.o
OBJS-$(CONFIG_H264CHROMA) += aarch64/h264chroma_init_aarch64.o
OBJS-$(CONFIG_H264DSP) += aarch64/h264dsp_init_aarch64.o
OBJS-$(CONFIG_H264PRED) += aarch64/h264pred_init.o
OBJS-$(CONFIG_H264QPEL) += aarch64/h264qpel_init_aarch64.o
OBJS-$(CONFIG_HPELDSP) += aarch64/hpeldsp_init_aarch64.o
OBJS-$(CONFIG_IDCTDSP) += aarch64/idctdsp_init_aarch64.o
OBJS-$(CONFIG_ME_CMP) += aarch64/me_cmp_init_aarch64.o
OBJS-$(CONFIG_MPEGAUDIODSP) += aarch64/mpegaudiodsp_init.o
OBJS-$(CONFIG_MPEGVIDEOENC) += aarch64/mpegvideoencdsp_init.o
OBJS-$(CONFIG_NEON_CLOBBER_TEST) += aarch64/neontest.o
OBJS-$(CONFIG_PIXBLOCKDSP) += aarch64/pixblockdsp_init_aarch64.o
OBJS-$(CONFIG_VIDEODSP) += aarch64/videodsp_init.o
OBJS-$(CONFIG_VP8DSP) += aarch64/vp8dsp_init_aarch64.o
# decoders/encoders
OBJS-$(CONFIG_AAC_DECODER) += aarch64/aacpsdsp_init_aarch64.o \
aarch64/sbrdsp_init_aarch64.o
OBJS-$(CONFIG_DCA_DECODER) += aarch64/synth_filter_init.o
OBJS-$(CONFIG_OPUS_DECODER) += aarch64/opusdsp_init.o
OBJS-$(CONFIG_RV40_DECODER) += aarch64/rv40dsp_init_aarch64.o
OBJS-$(CONFIG_VC1DSP) += aarch64/vc1dsp_init_aarch64.o
OBJS-$(CONFIG_VORBIS_DECODER) += aarch64/vorbisdsp_init.o
OBJS-$(CONFIG_VP9_DECODER) += aarch64/vp9dsp_init_10bpp_aarch64.o \
aarch64/vp9dsp_init_12bpp_aarch64.o \
aarch64/vp9mc_aarch64.o \
aarch64/vp9dsp_init_aarch64.o
# ARMv8 optimizations
# subsystems
ARMV8-OBJS-$(CONFIG_VIDEODSP) += aarch64/videodsp.o
# NEON optimizations
# subsystems
NEON-OBJS-$(CONFIG_AAC_DECODER) += aarch64/sbrdsp_neon.o
NEON-OBJS-$(CONFIG_AC3DSP) += aarch64/ac3dsp_neon.o
NEON-OBJS-$(CONFIG_FDCTDSP) += aarch64/fdctdsp_neon.o
NEON-OBJS-$(CONFIG_FMTCONVERT) += aarch64/fmtconvert_neon.o
NEON-OBJS-$(CONFIG_H264CHROMA) += aarch64/h264cmc_neon.o
NEON-OBJS-$(CONFIG_H264DSP) += aarch64/h264dsp_neon.o \
aarch64/h264idct_neon.o
NEON-OBJS-$(CONFIG_H264PRED) += aarch64/h264pred_neon.o
NEON-OBJS-$(CONFIG_H264QPEL) += aarch64/h264qpel_neon.o \
aarch64/hpeldsp_neon.o
NEON-OBJS-$(CONFIG_HPELDSP) += aarch64/hpeldsp_neon.o
NEON-OBJS-$(CONFIG_IDCTDSP) += aarch64/idctdsp_neon.o \
aarch64/simple_idct_neon.o
NEON-OBJS-$(CONFIG_ME_CMP) += aarch64/me_cmp_neon.o
NEON-OBJS-$(CONFIG_MPEGAUDIODSP) += aarch64/mpegaudiodsp_neon.o
NEON-OBJS-$(CONFIG_MPEGVIDEOENC) += aarch64/mpegvideoencdsp_neon.o
NEON-OBJS-$(CONFIG_PIXBLOCKDSP) += aarch64/pixblockdsp_neon.o
NEON-OBJS-$(CONFIG_VC1DSP) += aarch64/vc1dsp_neon.o
NEON-OBJS-$(CONFIG_VP8DSP) += aarch64/vp8dsp_neon.o
# decoders/encoders
NEON-OBJS-$(CONFIG_AAC_DECODER) += aarch64/aacpsdsp_neon.o
NEON-OBJS-$(CONFIG_DCA_DECODER) += aarch64/synth_filter_neon.o
NEON-OBJS-$(CONFIG_OPUS_DECODER) += aarch64/opusdsp_neon.o
NEON-OBJS-$(CONFIG_VORBIS_DECODER) += aarch64/vorbisdsp_neon.o
NEON-OBJS-$(CONFIG_VP9_DECODER) += aarch64/vp9itxfm_16bpp_neon.o \
aarch64/vp9itxfm_neon.o \
aarch64/vp9lpf_16bpp_neon.o \
aarch64/vp9lpf_neon.o \
aarch64/vp9mc_16bpp_neon.o \
aarch64/vp9mc_neon.o
NEON-OBJS-$(CONFIG_HEVC_DECODER) += aarch64/hevcdsp_deblock_neon.o \
aarch64/hevcdsp_idct_neon.o \
aarch64/hevcdsp_init_aarch64.o \
aarch64/h26x/epel_neon.o \
aarch64/h26x/qpel_neon.o \
aarch64/h26x/sao_neon.o
@@ -0,0 +1,49 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/aacpsdsp.h"
void ff_ps_add_squares_neon(float *dst, const float (*src)[2], int n);
void ff_ps_mul_pair_single_neon(float (*dst)[2], float (*src0)[2],
float *src1, int n);
void ff_ps_hybrid_analysis_neon(float (*out)[2], float (*in)[2],
const float (*filter)[8][2],
ptrdiff_t stride, int n);
void ff_ps_stereo_interpolate_neon(float (*l)[2], float (*r)[2],
float h[2][4], float h_step[2][4],
int len);
void ff_ps_stereo_interpolate_ipdopd_neon(float (*l)[2], float (*r)[2],
float h[2][4], float h_step[2][4],
int len);
av_cold void ff_psdsp_init_aarch64(PSDSPContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
s->add_squares = ff_ps_add_squares_neon;
s->mul_pair_single = ff_ps_mul_pair_single_neon;
s->hybrid_analysis = ff_ps_hybrid_analysis_neon;
s->stereo_interpolate[0] = ff_ps_stereo_interpolate_neon;
s->stereo_interpolate[1] = ff_ps_stereo_interpolate_ipdopd_neon;
}
}
+148
View File
@@ -0,0 +1,148 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
function ff_ps_add_squares_neon, export=1
1: ld1 {v0.4s,v1.4s}, [x1], #32
fmul v0.4s, v0.4s, v0.4s
fmul v1.4s, v1.4s, v1.4s
faddp v2.4s, v0.4s, v1.4s
ld1 {v3.4s}, [x0]
fadd v3.4s, v3.4s, v2.4s
st1 {v3.4s}, [x0], #16
subs w2, w2, #4
b.gt 1b
ret
endfunc
function ff_ps_mul_pair_single_neon, export=1
1: ld1 {v0.4s,v1.4s}, [x1], #32
ld1 {v2.4s}, [x2], #16
zip1 v3.4s, v2.4s, v2.4s
zip2 v4.4s, v2.4s, v2.4s
fmul v0.4s, v0.4s, v3.4s
fmul v1.4s, v1.4s, v4.4s
st1 {v0.4s,v1.4s}, [x0], #32
subs w3, w3, #4
b.gt 1b
ret
endfunc
function ff_ps_stereo_interpolate_neon, export=1
ld1 {v0.4s}, [x2]
ld1 {v1.4s}, [x3]
zip1 v4.4s, v0.4s, v0.4s
zip2 v5.4s, v0.4s, v0.4s
zip1 v6.4s, v1.4s, v1.4s
zip2 v7.4s, v1.4s, v1.4s
1: ld1 {v2.2s}, [x0]
ld1 {v3.2s}, [x1]
fadd v4.4s, v4.4s, v6.4s
fadd v5.4s, v5.4s, v7.4s
mov v2.d[1], v2.d[0]
mov v3.d[1], v3.d[0]
fmul v2.4s, v2.4s, v4.4s
fmla v2.4s, v3.4s, v5.4s
st1 {v2.d}[0], [x0], #8
st1 {v2.d}[1], [x1], #8
subs w4, w4, #1
b.gt 1b
ret
endfunc
function ff_ps_stereo_interpolate_ipdopd_neon, export=1
ld1 {v0.4s,v1.4s}, [x2]
ld1 {v6.4s,v7.4s}, [x3]
fneg v2.4s, v1.4s
fneg v3.4s, v7.4s
zip1 v16.4s, v0.4s, v0.4s
zip2 v17.4s, v0.4s, v0.4s
zip1 v18.4s, v2.4s, v1.4s
zip2 v19.4s, v2.4s, v1.4s
zip1 v20.4s, v6.4s, v6.4s
zip2 v21.4s, v6.4s, v6.4s
zip1 v22.4s, v3.4s, v7.4s
zip2 v23.4s, v3.4s, v7.4s
1: ld1 {v2.2s}, [x0]
ld1 {v3.2s}, [x1]
fadd v16.4s, v16.4s, v20.4s
fadd v17.4s, v17.4s, v21.4s
mov v2.d[1], v2.d[0]
mov v3.d[1], v3.d[0]
fmul v4.4s, v2.4s, v16.4s
fmla v4.4s, v3.4s, v17.4s
fadd v18.4s, v18.4s, v22.4s
fadd v19.4s, v19.4s, v23.4s
ext v2.16b, v2.16b, v2.16b, #4
ext v3.16b, v3.16b, v3.16b, #4
fmla v4.4s, v2.4s, v18.4s
fmla v4.4s, v3.4s, v19.4s
st1 {v4.d}[0], [x0], #8
st1 {v4.d}[1], [x1], #8
subs w4, w4, #1
b.gt 1b
ret
endfunc
function ff_ps_hybrid_analysis_neon, export=1
lsl x3, x3, #3
ld2 {v0.4s,v1.4s}, [x1], #32
ld2 {v2.2s,v3.2s}, [x1], #16
ld1 {v24.2s}, [x1], #8
ld2 {v4.2s,v5.2s}, [x1], #16
ld2 {v6.4s,v7.4s}, [x1]
rev64 v6.4s, v6.4s
rev64 v7.4s, v7.4s
ext v6.16b, v6.16b, v6.16b, #8
ext v7.16b, v7.16b, v7.16b, #8
rev64 v4.2s, v4.2s
rev64 v5.2s, v5.2s
mov v2.d[1], v3.d[0]
mov v4.d[1], v5.d[0]
mov v5.d[1], v2.d[0]
mov v3.d[1], v4.d[0]
fadd v16.4s, v0.4s, v6.4s
fadd v17.4s, v1.4s, v7.4s
fsub v18.4s, v1.4s, v7.4s
fsub v19.4s, v0.4s, v6.4s
fadd v22.4s, v2.4s, v4.4s
fsub v23.4s, v5.4s, v3.4s
trn1 v20.2d, v22.2d, v23.2d // {re4+re8, re5+re7, im8-im4, im7-im5}
trn2 v21.2d, v22.2d, v23.2d // {im4+im8, im5+im7, re4-re8, re5-re7}
1: ld2 {v2.4s,v3.4s}, [x2], #32
ld2 {v4.2s,v5.2s}, [x2], #16
ld1 {v6.2s}, [x2], #8
add x2, x2, #8
mov v4.d[1], v5.d[0]
mov v6.s[1], v6.s[0]
fmul v6.2s, v6.2s, v24.2s
fmul v0.4s, v2.4s, v16.4s
fmul v1.4s, v2.4s, v17.4s
fmls v0.4s, v3.4s, v18.4s
fmla v1.4s, v3.4s, v19.4s
fmla v0.4s, v4.4s, v20.4s
fmla v1.4s, v4.4s, v21.4s
faddp v0.4s, v0.4s, v1.4s
faddp v0.4s, v0.4s, v0.4s
fadd v0.2s, v0.2s, v6.2s
st1 {v0.2s}, [x0], x3
subs w4, w4, #1
b.gt 1b
ret
endfunc
@@ -0,0 +1,50 @@
/*
* Copyright (c) 2024 Geoff Hill <geoff@geoffhill.org>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with FFmpeg; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include <stdint.h>
#include "libavutil/arm/cpu.h"
#include "libavutil/attributes.h"
#include "libavcodec/ac3dsp.h"
#include "config.h"
void ff_ac3_exponent_min_neon(uint8_t *exp, int num_reuse_blocks, int nb_coefs);
void ff_ac3_extract_exponents_neon(uint8_t *exp, int32_t *coef, int nb_coefs);
void ff_float_to_fixed24_neon(int32_t *dst, const float *src, size_t len);
void ff_ac3_sum_square_butterfly_int32_neon(int64_t sum[4],
const int32_t *coef0,
const int32_t *coef1,
int len);
void ff_ac3_sum_square_butterfly_float_neon(float sum[4],
const float *coef0,
const float *coef1,
int len);
av_cold void ff_ac3dsp_init_aarch64(AC3DSPContext *c)
{
int cpu_flags = av_get_cpu_flags();
if (!have_neon(cpu_flags)) return;
c->ac3_exponent_min = ff_ac3_exponent_min_neon;
c->extract_exponents = ff_ac3_extract_exponents_neon;
c->float_to_fixed24 = ff_float_to_fixed24_neon;
c->sum_square_butterfly_int32 = ff_ac3_sum_square_butterfly_int32_neon;
c->sum_square_butterfly_float = ff_ac3_sum_square_butterfly_float_neon;
}
+111
View File
@@ -0,0 +1,111 @@
/*
* Copyright (c) 2011 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2024 Geoff Hill <geoff@geoffhill.org>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with FFmpeg; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libavutil/aarch64/asm.S"
function ff_ac3_exponent_min_neon, export=1
cbz w1, 3f
1: ld1 {v0.16b}, [x0]
mov w3, w1
add x4, x0, #256
2: ld1 {v1.16b}, [x4]
umin v0.16b, v0.16b, v1.16b
add x4, x4, #256
subs w3, w3, #1
b.gt 2b
st1 {v0.16b}, [x0], #16
subs w2, w2, #16
b.gt 1b
3: ret
endfunc
function ff_ac3_extract_exponents_neon, export=1
movi v1.4s, #8
1: ld1 {v0.4s}, [x1], #16
abs v0.4s, v0.4s
clz v0.4s, v0.4s
sub v0.4s, v0.4s, v1.4s
xtn v0.4h, v0.4s
xtn v0.8b, v0.8h
st1 {v0.s}[0], [x0], #4
subs w2, w2, #4
b.gt 1b
ret
endfunc
function ff_float_to_fixed24_neon, export=1
1: ld1 {v0.4s, v1.4s}, [x1], #32
fcvtzs v0.4s, v0.4s, #24
ld1 {v2.4s, v3.4s}, [x1], #32
fcvtzs v1.4s, v1.4s, #24
fcvtzs v2.4s, v2.4s, #24
st1 {v0.4s, v1.4s}, [x0], #32
fcvtzs v3.4s, v3.4s, #24
st1 {v2.4s, v3.4s}, [x0], #32
subs w2, w2, #16
b.ne 1b
ret
endfunc
function ff_ac3_sum_square_butterfly_int32_neon, export=1
movi v0.2d, #0
movi v1.2d, #0
movi v2.2d, #0
movi v3.2d, #0
1: ld1 {v4.2s}, [x1], #8
ld1 {v5.2s}, [x2], #8
add v6.2s, v4.2s, v5.2s
sub v7.2s, v4.2s, v5.2s
smlal v0.2d, v4.2s, v4.2s
smlal v1.2d, v5.2s, v5.2s
smlal v2.2d, v6.2s, v6.2s
smlal v3.2d, v7.2s, v7.2s
subs w3, w3, #2
b.gt 1b
addp d0, v0.2d
addp d1, v1.2d
addp d2, v2.2d
addp d3, v3.2d
st1 {v0.1d-v3.1d}, [x0]
ret
endfunc
function ff_ac3_sum_square_butterfly_float_neon, export=1
movi v0.4s, #0
movi v1.4s, #0
movi v2.4s, #0
movi v3.4s, #0
1: ld1 {v30.4s}, [x1], #16
ld1 {v31.4s}, [x2], #16
fadd v16.4s, v30.4s, v31.4s
fsub v17.4s, v30.4s, v31.4s
fmla v0.4s, v30.4s, v30.4s
fmla v1.4s, v31.4s, v31.4s
fmla v2.4s, v16.4s, v16.4s
fmla v3.4s, v17.4s, v17.4s
subs w3, w3, #4
b.gt 1b
faddp v0.4s, v0.4s, v1.4s
faddp v2.4s, v2.4s, v3.4s
faddp v0.4s, v0.4s, v2.4s
st1 {v0.4s}, [x0]
ret
endfunc
+104
View File
@@ -0,0 +1,104 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_AARCH64_CABAC_H
#define AVCODEC_AARCH64_CABAC_H
#include "config.h"
#if HAVE_INLINE_ASM
#include "libavutil/attributes.h"
#include "libavutil/internal.h"
#include "libavcodec/cabac.h"
#define get_cabac_inline get_cabac_inline_aarch64
static av_always_inline int get_cabac_inline_aarch64(CABACContext *c,
uint8_t *const state)
{
int bit;
void *reg_a, *reg_b, *reg_c, *tmp;
__asm__ volatile(
"ldrb %w[bit] , [%[state]] \n\t"
"add %[r_b] , %[tables] , %[lps_off] \n\t"
"mov %w[tmp] , %w[range] \n\t"
"and %w[range] , %w[range] , #0xC0 \n\t"
"lsl %w[r_c] , %w[range] , #1 \n\t"
"add %[r_b] , %[r_b] , %w[bit], UXTW \n\t"
"ldrb %w[range] , [%[r_b], %w[r_c], SXTW] \n\t"
"sub %w[r_c] , %w[tmp] , %w[range] \n\t"
"lsl %w[tmp] , %w[r_c] , #17 \n\t"
"cmp %w[tmp] , %w[low] \n\t"
"csel %w[tmp] , %w[tmp] , wzr , cc \n\t"
"csel %w[range] , %w[r_c] , %w[range], gt \n\t"
"cinv %w[bit] , %w[bit] , cc \n\t"
"sub %w[low] , %w[low] , %w[tmp] \n\t"
"add %[r_b] , %[tables] , %[norm_off] \n\t"
"add %[r_a] , %[tables] , %[mlps_off] \n\t"
"ldrb %w[tmp] , [%[r_b], %w[range], SXTW] \n\t"
"ldrb %w[r_a] , [%[r_a], %w[bit], SXTW] \n\t"
"lsl %w[low] , %w[low] , %w[tmp] \n\t"
"lsl %w[range] , %w[range] , %w[tmp] \n\t"
"uxth %w[r_c] , %w[low] \n\t"
"strb %w[r_a] , [%[state]] \n\t"
"cbnz %w[r_c] , 2f \n\t"
"ldr %[r_c] , [%[c], %[byte]] \n\t"
"ldr %[r_a] , [%[c], %[end]] \n\t"
"ldrh %w[tmp] , [%[r_c]] \n\t"
"cmp %[r_c] , %[r_a] \n\t"
"b.ge 1f \n\t"
"add %[r_a] , %[r_c] , #2 \n\t"
"str %[r_a] , [%[c], %[byte]] \n\t"
"1: \n\t"
"sub %w[r_c] , %w[low] , #1 \n\t"
"eor %w[r_c] , %w[r_c] , %w[low] \n\t"
"rev %w[tmp] , %w[tmp] \n\t"
"lsr %w[r_c] , %w[r_c] , #15 \n\t"
"lsr %w[tmp] , %w[tmp] , #15 \n\t"
"ldrb %w[r_c] , [%[r_b], %w[r_c], SXTW] \n\t"
"mov %w[r_b] , #0xFFFF \n\t"
"mov %w[r_a] , #7 \n\t"
"sub %w[tmp] , %w[tmp] , %w[r_b] \n\t"
"sub %w[r_c] , %w[r_a] , %w[r_c] \n\t"
"lsl %w[tmp] , %w[tmp] , %w[r_c] \n\t"
"add %w[low] , %w[low] , %w[tmp] \n\t"
"2: \n\t"
: [bit]"=&r"(bit),
[low]"+&r"(c->low),
[range]"+&r"(c->range),
[r_a]"=&r"(reg_a),
[r_b]"=&r"(reg_b),
[r_c]"=&r"(reg_c),
[tmp]"=&r"(tmp)
: [c]"r"(c),
[state]"r"(state),
[tables]"r"(ff_h264_cabac_tables),
[byte]"i"(offsetof(CABACContext, bytestream)),
[end]"i"(offsetof(CABACContext, bytestream_end)),
[norm_off]"I"(H264_NORM_SHIFT_OFFSET),
[lps_off]"I"(H264_LPS_RANGE_OFFSET),
[mlps_off]"I"(H264_MLPS_STATE_OFFSET + 128)
: "memory", "cc"
);
return bit & 1;
}
#endif /* HAVE_INLINE_ASM */
#endif /* AVCODEC_AARCH64_CABAC_H */
+26
View File
@@ -0,0 +1,26 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_AARCH64_FDCT_H
#define AVCODEC_AARCH64_FDCT_H
#include <stdint.h>
void ff_fdct_neon(int16_t *block);
#endif /* AVCODEC_AARCH64_FDCT_H */
@@ -0,0 +1,39 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/fdctdsp.h"
#include "fdct.h"
av_cold void ff_fdctdsp_init_aarch64(FDCTDSPContext *c, AVCodecContext *avctx,
unsigned high_bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
if (!high_bit_depth) {
if (avctx->dct_algo == FF_DCT_AUTO ||
avctx->dct_algo == FF_DCT_NEON) {
c->fdct = ff_fdct_neon;
}
}
}
}
+368
View File
@@ -0,0 +1,368 @@
/*
* Armv8 Neon optimizations for libjpeg-turbo
*
* Copyright (C) 2009-2011, Nokia Corporation and/or its subsidiary(-ies).
* All Rights Reserved.
* Author: Siarhei Siamashka <siarhei.siamashka@nokia.com>
* Copyright (C) 2013-2014, Linaro Limited. All Rights Reserved.
* Author: Ragesh Radhakrishnan <ragesh.r@linaro.org>
* Copyright (C) 2014-2016, 2020, D. R. Commander. All Rights Reserved.
* Copyright (C) 2015-2016, 2018, Matthieu Darbois. All Rights Reserved.
* Copyright (C) 2016, Siarhei Siamashka. All Rights Reserved.
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
* 3. This notice may not be removed or altered from any source distribution.
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
// #define EIGHT_BIT_SAMPLES
/* Constants for jsimd_fdct_islow_neon() */
#define F_0_298 2446 /* FIX(0.298631336) */
#define F_0_390 3196 /* FIX(0.390180644) */
#define F_0_541 4433 /* FIX(0.541196100) */
#define F_0_765 6270 /* FIX(0.765366865) */
#define F_0_899 7373 /* FIX(0.899976223) */
#define F_1_175 9633 /* FIX(1.175875602) */
#define F_1_501 12299 /* FIX(1.501321110) */
#define F_1_847 15137 /* FIX(1.847759065) */
#define F_1_961 16069 /* FIX(1.961570560) */
#define F_2_053 16819 /* FIX(2.053119869) */
#define F_2_562 20995 /* FIX(2.562915447) */
#define F_3_072 25172 /* FIX(3.072711026) */
const jsimd_fdct_islow_neon_consts, align=4
.short F_0_298
.short -F_0_390
.short F_0_541
.short F_0_765
.short - F_0_899
.short F_1_175
.short F_1_501
.short - F_1_847
.short - F_1_961
.short F_2_053
.short - F_2_562
.short F_3_072
.short 0 /* padding */
.short 0
.short 0
.short 0
endconst
#undef F_0_298
#undef F_0_390
#undef F_0_541
#undef F_0_765
#undef F_0_899
#undef F_1_175
#undef F_1_501
#undef F_1_847
#undef F_1_961
#undef F_2_053
#undef F_2_562
#undef F_3_072
/*****************************************************************************/
/*
* jsimd_fdct_islow_neon
*
* This file contains a slower but more accurate integer implementation of the
* forward DCT (Discrete Cosine Transform). The following code is based
* directly on the IJG''s original jfdctint.c; see the jfdctint.c for
* more details.
*/
#define CONST_BITS 13
#ifdef EIGHT_BIT_SAMPLES
#define PASS1_BITS 2
#else
#define PASS1_BITS 1 /* lose a little precision to avoid overflow */
#endif
#define DESCALE_P1 (CONST_BITS - PASS1_BITS)
#define DESCALE_P2 (CONST_BITS + PASS1_BITS)
#define XFIX_P_0_298 v0.h[0]
#define XFIX_N_0_390 v0.h[1]
#define XFIX_P_0_541 v0.h[2]
#define XFIX_P_0_765 v0.h[3]
#define XFIX_N_0_899 v0.h[4]
#define XFIX_P_1_175 v0.h[5]
#define XFIX_P_1_501 v0.h[6]
#define XFIX_N_1_847 v0.h[7]
#define XFIX_N_1_961 v1.h[0]
#define XFIX_P_2_053 v1.h[1]
#define XFIX_N_2_562 v1.h[2]
#define XFIX_P_3_072 v1.h[3]
function ff_fdct_neon, export=1
DATA .req x0
TMP .req x9
/* Load constants */
movrel TMP, jsimd_fdct_islow_neon_consts
ld1 {v0.8h, v1.8h}, [TMP]
/* Load all DATA into Neon registers with the following allocation:
* 0 1 2 3 | 4 5 6 7
* ---------+--------
* 0 | d16 | d17 | v16.8h
* 1 | d18 | d19 | v17.8h
* 2 | d20 | d21 | v18.8h
* 3 | d22 | d23 | v19.8h
* 4 | d24 | d25 | v20.8h
* 5 | d26 | d27 | v21.8h
* 6 | d28 | d29 | v22.8h
* 7 | d30 | d31 | v23.8h
*/
ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64
ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA]
sub DATA, DATA, #64
/* Transpose */
transpose_8x8H v16, v17, v18, v19, v20, v21, v22, v23, v31, v2
/* 1-D FDCT */
add v24.8h, v16.8h, v23.8h /* tmp0 = dataptr[0] + dataptr[7]; */
sub v31.8h, v16.8h, v23.8h /* tmp7 = dataptr[0] - dataptr[7]; */
add v25.8h, v17.8h, v22.8h /* tmp1 = dataptr[1] + dataptr[6]; */
sub v30.8h, v17.8h, v22.8h /* tmp6 = dataptr[1] - dataptr[6]; */
add v26.8h, v18.8h, v21.8h /* tmp2 = dataptr[2] + dataptr[5]; */
sub v29.8h, v18.8h, v21.8h /* tmp5 = dataptr[2] - dataptr[5]; */
add v27.8h, v19.8h, v20.8h /* tmp3 = dataptr[3] + dataptr[4]; */
sub v28.8h, v19.8h, v20.8h /* tmp4 = dataptr[3] - dataptr[4]; */
/* Even part */
add v4.8h, v24.8h, v27.8h /* tmp10 = tmp0 + tmp3; */
sub v5.8h, v24.8h, v27.8h /* tmp13 = tmp0 - tmp3; */
add v6.8h, v25.8h, v26.8h /* tmp11 = tmp1 + tmp2; */
sub v7.8h, v25.8h, v26.8h /* tmp12 = tmp1 - tmp2; */
add v16.8h, v4.8h, v6.8h /* tmp10 + tmp11 */
sub v20.8h, v4.8h, v6.8h /* tmp10 - tmp11 */
add v18.8h, v7.8h, v5.8h /* tmp12 + tmp13 */
shl v16.8h, v16.8h, #PASS1_BITS /* dataptr[0] = (DCTELEM)LEFT_SHIFT(tmp10 + tmp11, PASS1_BITS); */
shl v20.8h, v20.8h, #PASS1_BITS /* dataptr[4] = (DCTELEM)LEFT_SHIFT(tmp10 - tmp11, PASS1_BITS); */
smull2 v24.4s, v18.8h, XFIX_P_0_541 /* z1 hi = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1 lo = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
mov v22.16b, v18.16b
mov v25.16b, v24.16b
smlal v18.4s, v5.4h, XFIX_P_0_765 /* lo z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
smlal2 v24.4s, v5.8h, XFIX_P_0_765 /* hi z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
smlal v22.4s, v7.4h, XFIX_N_1_847 /* lo z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
smlal2 v25.4s, v7.8h, XFIX_N_1_847 /* hi z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
rshrn v18.4h, v18.4s, #DESCALE_P1
rshrn v22.4h, v22.4s, #DESCALE_P1
rshrn2 v18.8h, v24.4s, #DESCALE_P1 /* dataptr[2] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp13, XFIX_P_0_765), CONST_BITS-PASS1_BITS); */
rshrn2 v22.8h, v25.4s, #DESCALE_P1 /* dataptr[6] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp12, XFIX_N_1_847), CONST_BITS-PASS1_BITS); */
/* Odd part */
add v2.8h, v28.8h, v31.8h /* z1 = tmp4 + tmp7; */
add v3.8h, v29.8h, v30.8h /* z2 = tmp5 + tmp6; */
add v6.8h, v28.8h, v30.8h /* z3 = tmp4 + tmp6; */
add v7.8h, v29.8h, v31.8h /* z4 = tmp5 + tmp7; */
smull v4.4s, v6.4h, XFIX_P_1_175 /* z5 lo = z3 lo * XFIX_P_1_175 */
smull2 v5.4s, v6.8h, XFIX_P_1_175
smlal v4.4s, v7.4h, XFIX_P_1_175 /* z5 = MULTIPLY(z3 + z4, FIX_1_175875602); */
smlal2 v5.4s, v7.8h, XFIX_P_1_175
smull2 v24.4s, v28.8h, XFIX_P_0_298
smull2 v25.4s, v29.8h, XFIX_P_2_053
smull2 v26.4s, v30.8h, XFIX_P_3_072
smull2 v27.4s, v31.8h, XFIX_P_1_501
smull v23.4s, v28.4h, XFIX_P_0_298 /* tmp4 = MULTIPLY(tmp4, FIX_0_298631336); */
smull v21.4s, v29.4h, XFIX_P_2_053 /* tmp5 = MULTIPLY(tmp5, FIX_2_053119869); */
smull v19.4s, v30.4h, XFIX_P_3_072 /* tmp6 = MULTIPLY(tmp6, FIX_3_072711026); */
smull v17.4s, v31.4h, XFIX_P_1_501 /* tmp7 = MULTIPLY(tmp7, FIX_1_501321110); */
smull2 v28.4s, v2.8h, XFIX_N_0_899
smull2 v29.4s, v3.8h, XFIX_N_2_562
smull2 v30.4s, v6.8h, XFIX_N_1_961
smull2 v31.4s, v7.8h, XFIX_N_0_390
smull v2.4s, v2.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, -FIX_0_899976223); */
smull v3.4s, v3.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, -FIX_2_562915447); */
smull v6.4s, v6.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, -FIX_1_961570560); */
smull v7.4s, v7.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, -FIX_0_390180644); */
add v6.4s, v6.4s, v4.4s /* z3 += z5 */
add v30.4s, v30.4s, v5.4s
add v7.4s, v7.4s, v4.4s /* z4 += z5 */
add v31.4s, v31.4s, v5.4s
add v23.4s, v23.4s, v2.4s /* tmp4 += z1 */
add v24.4s, v24.4s, v28.4s
add v21.4s, v21.4s, v3.4s /* tmp5 += z2 */
add v25.4s, v25.4s, v29.4s
add v19.4s, v19.4s, v6.4s /* tmp6 += z3 */
add v26.4s, v26.4s, v30.4s
add v17.4s, v17.4s, v7.4s /* tmp7 += z4 */
add v27.4s, v27.4s, v31.4s
add v23.4s, v23.4s, v6.4s /* tmp4 += z3 */
add v24.4s, v24.4s, v30.4s
add v21.4s, v21.4s, v7.4s /* tmp5 += z4 */
add v25.4s, v25.4s, v31.4s
add v19.4s, v19.4s, v3.4s /* tmp6 += z2 */
add v26.4s, v26.4s, v29.4s
add v17.4s, v17.4s, v2.4s /* tmp7 += z1 */
add v27.4s, v27.4s, v28.4s
rshrn v23.4h, v23.4s, #DESCALE_P1
rshrn v21.4h, v21.4s, #DESCALE_P1
rshrn v19.4h, v19.4s, #DESCALE_P1
rshrn v17.4h, v17.4s, #DESCALE_P1
rshrn2 v23.8h, v24.4s, #DESCALE_P1 /* dataptr[7] = (DCTELEM)DESCALE(tmp4 + z1 + z3, CONST_BITS-PASS1_BITS); */
rshrn2 v21.8h, v25.4s, #DESCALE_P1 /* dataptr[5] = (DCTELEM)DESCALE(tmp5 + z2 + z4, CONST_BITS-PASS1_BITS); */
rshrn2 v19.8h, v26.4s, #DESCALE_P1 /* dataptr[3] = (DCTELEM)DESCALE(tmp6 + z2 + z3, CONST_BITS-PASS1_BITS); */
rshrn2 v17.8h, v27.4s, #DESCALE_P1 /* dataptr[1] = (DCTELEM)DESCALE(tmp7 + z1 + z4, CONST_BITS-PASS1_BITS); */
/* Transpose */
transpose_8x8H v16, v17, v18, v19, v20, v21, v22, v23, v31, v2
/* 1-D FDCT */
add v24.8h, v16.8h, v23.8h /* tmp0 = dataptr[0] + dataptr[7]; */
sub v31.8h, v16.8h, v23.8h /* tmp7 = dataptr[0] - dataptr[7]; */
add v25.8h, v17.8h, v22.8h /* tmp1 = dataptr[1] + dataptr[6]; */
sub v30.8h, v17.8h, v22.8h /* tmp6 = dataptr[1] - dataptr[6]; */
add v26.8h, v18.8h, v21.8h /* tmp2 = dataptr[2] + dataptr[5]; */
sub v29.8h, v18.8h, v21.8h /* tmp5 = dataptr[2] - dataptr[5]; */
add v27.8h, v19.8h, v20.8h /* tmp3 = dataptr[3] + dataptr[4]; */
sub v28.8h, v19.8h, v20.8h /* tmp4 = dataptr[3] - dataptr[4]; */
/* Even part */
add v4.8h, v24.8h, v27.8h /* tmp10 = tmp0 + tmp3; */
sub v5.8h, v24.8h, v27.8h /* tmp13 = tmp0 - tmp3; */
add v6.8h, v25.8h, v26.8h /* tmp11 = tmp1 + tmp2; */
sub v7.8h, v25.8h, v26.8h /* tmp12 = tmp1 - tmp2; */
add v16.8h, v4.8h, v6.8h /* tmp10 + tmp11 */
sub v20.8h, v4.8h, v6.8h /* tmp10 - tmp11 */
add v18.8h, v7.8h, v5.8h /* tmp12 + tmp13 */
srshr v16.8h, v16.8h, #PASS1_BITS /* dataptr[0] = (DCTELEM)DESCALE(tmp10 + tmp11, PASS1_BITS); */
srshr v20.8h, v20.8h, #PASS1_BITS /* dataptr[4] = (DCTELEM)DESCALE(tmp10 - tmp11, PASS1_BITS); */
smull2 v24.4s, v18.8h, XFIX_P_0_541 /* z1 hi = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1 lo = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
mov v22.16b, v18.16b
mov v25.16b, v24.16b
smlal v18.4s, v5.4h, XFIX_P_0_765 /* lo z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
smlal2 v24.4s, v5.8h, XFIX_P_0_765 /* hi z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
smlal v22.4s, v7.4h, XFIX_N_1_847 /* lo z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
smlal2 v25.4s, v7.8h, XFIX_N_1_847 /* hi z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
rshrn v18.4h, v18.4s, #DESCALE_P2
rshrn v22.4h, v22.4s, #DESCALE_P2
rshrn2 v18.8h, v24.4s, #DESCALE_P2 /* dataptr[2] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp13, XFIX_P_0_765), CONST_BITS+PASS1_BITS); */
rshrn2 v22.8h, v25.4s, #DESCALE_P2 /* dataptr[6] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp12, XFIX_N_1_847), CONST_BITS+PASS1_BITS); */
/* Odd part */
add v2.8h, v28.8h, v31.8h /* z1 = tmp4 + tmp7; */
add v3.8h, v29.8h, v30.8h /* z2 = tmp5 + tmp6; */
add v6.8h, v28.8h, v30.8h /* z3 = tmp4 + tmp6; */
add v7.8h, v29.8h, v31.8h /* z4 = tmp5 + tmp7; */
smull v4.4s, v6.4h, XFIX_P_1_175 /* z5 lo = z3 lo * XFIX_P_1_175 */
smull2 v5.4s, v6.8h, XFIX_P_1_175
smlal v4.4s, v7.4h, XFIX_P_1_175 /* z5 = MULTIPLY(z3 + z4, FIX_1_175875602); */
smlal2 v5.4s, v7.8h, XFIX_P_1_175
smull2 v24.4s, v28.8h, XFIX_P_0_298
smull2 v25.4s, v29.8h, XFIX_P_2_053
smull2 v26.4s, v30.8h, XFIX_P_3_072
smull2 v27.4s, v31.8h, XFIX_P_1_501
smull v23.4s, v28.4h, XFIX_P_0_298 /* tmp4 = MULTIPLY(tmp4, FIX_0_298631336); */
smull v21.4s, v29.4h, XFIX_P_2_053 /* tmp5 = MULTIPLY(tmp5, FIX_2_053119869); */
smull v19.4s, v30.4h, XFIX_P_3_072 /* tmp6 = MULTIPLY(tmp6, FIX_3_072711026); */
smull v17.4s, v31.4h, XFIX_P_1_501 /* tmp7 = MULTIPLY(tmp7, FIX_1_501321110); */
smull2 v28.4s, v2.8h, XFIX_N_0_899
smull2 v29.4s, v3.8h, XFIX_N_2_562
smull2 v30.4s, v6.8h, XFIX_N_1_961
smull2 v31.4s, v7.8h, XFIX_N_0_390
smull v2.4s, v2.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, -FIX_0_899976223); */
smull v3.4s, v3.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, -FIX_2_562915447); */
smull v6.4s, v6.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, -FIX_1_961570560); */
smull v7.4s, v7.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, -FIX_0_390180644); */
add v6.4s, v6.4s, v4.4s /* z3 += z5 */
add v30.4s, v30.4s, v5.4s
add v7.4s, v7.4s, v4.4s /* z4 += z5 */
add v31.4s, v31.4s, v5.4s
add v23.4s, v23.4s, v2.4s /* tmp4 += z1 */
add v24.4s, v24.4s, v28.4s
add v21.4s, v21.4s, v3.4s /* tmp5 += z2 */
add v25.4s, v25.4s, v29.4s
add v19.4s, v19.4s, v6.4s /* tmp6 += z3 */
add v26.4s, v26.4s, v30.4s
add v17.4s, v17.4s, v7.4s /* tmp7 += z4 */
add v27.4s, v27.4s, v31.4s
add v23.4s, v23.4s, v6.4s /* tmp4 += z3 */
add v24.4s, v24.4s, v30.4s
add v21.4s, v21.4s, v7.4s /* tmp5 += z4 */
add v25.4s, v25.4s, v31.4s
add v19.4s, v19.4s, v3.4s /* tmp6 += z2 */
add v26.4s, v26.4s, v29.4s
add v17.4s, v17.4s, v2.4s /* tmp7 += z1 */
add v27.4s, v27.4s, v28.4s
rshrn v23.4h, v23.4s, #DESCALE_P2
rshrn v21.4h, v21.4s, #DESCALE_P2
rshrn v19.4h, v19.4s, #DESCALE_P2
rshrn v17.4h, v17.4s, #DESCALE_P2
rshrn2 v23.8h, v24.4s, #DESCALE_P2 /* dataptr[7] = (DCTELEM)DESCALE(tmp4 + z1 + z3, CONST_BITS+PASS1_BITS); */
rshrn2 v21.8h, v25.4s, #DESCALE_P2 /* dataptr[5] = (DCTELEM)DESCALE(tmp5 + z2 + z4, CONST_BITS+PASS1_BITS); */
rshrn2 v19.8h, v26.4s, #DESCALE_P2 /* dataptr[3] = (DCTELEM)DESCALE(tmp6 + z2 + z3, CONST_BITS+PASS1_BITS); */
rshrn2 v17.8h, v27.4s, #DESCALE_P2 /* dataptr[1] = (DCTELEM)DESCALE(tmp7 + z1 + z4, CONST_BITS+PASS1_BITS); */
/* Store results */
st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64
st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA]
ret
.unreq DATA
.unreq TMP
endfunc
#undef XFIX_P_0_298
#undef XFIX_N_0_390
#undef XFIX_P_0_541
#undef XFIX_P_0_765
#undef XFIX_N_0_899
#undef XFIX_P_1_175
#undef XFIX_P_1_501
#undef XFIX_N_1_847
#undef XFIX_N_1_961
#undef XFIX_P_2_053
#undef XFIX_N_2_562
#undef XFIX_P_3_072
@@ -0,0 +1,41 @@
/*
* ARM optimized Format Conversion Utils
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/fmtconvert.h"
void ff_int32_to_float_fmul_array8_neon(FmtConvertContext *c, float *dst,
const int32_t *src, const float *mul,
int len);
void ff_int32_to_float_fmul_scalar_neon(float *dst, const int32_t *src,
float mul, int len);
av_cold void ff_fmt_convert_init_aarch64(FmtConvertContext *c)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
c->int32_to_float_fmul_array8 = ff_int32_to_float_fmul_array8_neon;
c->int32_to_float_fmul_scalar = ff_int32_to_float_fmul_scalar_neon;
}
}
@@ -0,0 +1,76 @@
/*
* ARM NEON optimised Format Conversion Utils
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2015 Janne Grunau <janne-libav@jannau.net>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/aarch64/asm.S"
function ff_int32_to_float_fmul_scalar_neon, export=1
ld1 {v1.4s,v2.4s}, [x1], #32
scvtf v1.4s, v1.4s
scvtf v2.4s, v2.4s
1:
subs w2, w2, #8
fmul v3.4s, v1.4s, v0.s[0]
fmul v4.4s, v2.4s, v0.s[0]
b.le 2f
ld1 {v1.4s,v2.4s}, [x1], #32
st1 {v3.4s,v4.4s}, [x0], #32
scvtf v1.4s, v1.4s
scvtf v2.4s, v2.4s
b 1b
2:
st1 {v3.4s,v4.4s}, [x0]
ret
endfunc
function ff_int32_to_float_fmul_array8_neon, export=1
lsr w4, w4, #3
subs w5, w4, #1
b.eq 1f
2:
ld1 {v0.4s,v1.4s}, [x2], #32
ld1 {v2.4s,v3.4s}, [x2], #32
scvtf v0.4s, v0.4s
scvtf v1.4s, v1.4s
ld1 {v16.2s}, [x3], #8
scvtf v2.4s, v2.4s
scvtf v3.4s, v3.4s
fmul v4.4s, v0.4s, v16.s[0]
fmul v5.4s, v1.4s, v16.s[0]
fmul v6.4s, v2.4s, v16.s[1]
fmul v7.4s, v3.4s, v16.s[1]
st1 {v4.4s,v5.4s}, [x1], #32
st1 {v6.4s,v7.4s}, [x1], #32
subs w5, w5, #2
b.gt 2b
b.eq 1f
ret
1:
ld1 {v0.4s,v1.4s}, [x2]
ld1 {v16.s}[0], [x3]
scvtf v0.4s, v0.4s
scvtf v1.4s, v1.4s
fmul v4.4s, v0.4s, v16.s[0]
fmul v5.4s, v1.4s, v16.s[0]
st1 {v4.4s,v5.4s}, [x1]
ret
endfunc
@@ -0,0 +1,59 @@
/*
* ARM NEON optimised H.264 chroma functions
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/h264chroma.h"
#include "config.h"
void ff_put_h264_chroma_mc8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_put_h264_chroma_mc4_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_put_h264_chroma_mc2_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_h264_chroma_mc8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_h264_chroma_mc4_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_h264_chroma_mc2_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
av_cold void ff_h264chroma_init_aarch64(H264ChromaContext *c, int bit_depth)
{
const int high_bit_depth = bit_depth > 8;
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags) && !high_bit_depth) {
c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_neon;
c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_neon;
c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_neon;
c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_neon;
c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_neon;
c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_neon;
}
}
+452
View File
@@ -0,0 +1,452 @@
/*
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2013 Janne Grunau <janne-libav@jannau.net>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config_components.h"
#include "libavutil/aarch64/asm.S"
/* chroma_mc8(uint8_t *dst, const uint8_t *src, ptrdiff_t stride, int h, int x, int y) */
.macro h264_chroma_mc8 type, codec=h264
function ff_\type\()_\codec\()_chroma_mc8_neon, export=1
.ifc \type,avg
mov x8, x0
.endif
prfm pldl1strm, [x1]
prfm pldl1strm, [x1, x2]
.ifc \codec,rv40
movrel x6, rv40bias
lsr w9, w5, #1
lsr w10, w4, #1
lsl w9, w9, #3
lsl w10, w10, #1
add w9, w9, w10
add x6, x6, w9, uxtw
ld1r {v22.8h}, [x6]
.endif
.ifc \codec,vc1
movi v22.8h, #28
.endif
mul w7, w4, w5
lsl w14, w5, #3
lsl w13, w4, #3
cmp w7, #0
sub w6, w14, w7
sub w12, w13, w7
sub w4, w7, w13
sub w4, w4, w14
add w4, w4, #64
b.eq 2f
dup v0.8b, w4
dup v1.8b, w12
ld1 {v4.8b, v5.8b}, [x1], x2
dup v2.8b, w6
dup v3.8b, w7
ext v5.8b, v4.8b, v5.8b, #1
1: ld1 {v6.8b, v7.8b}, [x1], x2
umull v16.8h, v4.8b, v0.8b
umlal v16.8h, v5.8b, v1.8b
ext v7.8b, v6.8b, v7.8b, #1
ld1 {v4.8b, v5.8b}, [x1], x2
umlal v16.8h, v6.8b, v2.8b
prfm pldl1strm, [x1]
ext v5.8b, v4.8b, v5.8b, #1
umlal v16.8h, v7.8b, v3.8b
umull v17.8h, v6.8b, v0.8b
subs w3, w3, #2
umlal v17.8h, v7.8b, v1.8b
umlal v17.8h, v4.8b, v2.8b
umlal v17.8h, v5.8b, v3.8b
prfm pldl1strm, [x1, x2]
.ifc \codec,h264
rshrn v16.8b, v16.8h, #6
rshrn v17.8b, v17.8h, #6
.else
add v16.8h, v16.8h, v22.8h
add v17.8h, v17.8h, v22.8h
shrn v16.8b, v16.8h, #6
shrn v17.8b, v17.8h, #6
.endif
.ifc \type,avg
ld1 {v20.8b}, [x8], x2
ld1 {v21.8b}, [x8], x2
urhadd v16.8b, v16.8b, v20.8b
urhadd v17.8b, v17.8b, v21.8b
.endif
st1 {v16.8b}, [x0], x2
st1 {v17.8b}, [x0], x2
b.gt 1b
ret
2: adds w12, w12, w6
dup v0.8b, w4
b.eq 5f
tst w6, w6
dup v1.8b, w12
b.eq 4f
ld1 {v4.8b}, [x1], x2
3: ld1 {v6.8b}, [x1], x2
umull v16.8h, v4.8b, v0.8b
umlal v16.8h, v6.8b, v1.8b
ld1 {v4.8b}, [x1], x2
umull v17.8h, v6.8b, v0.8b
umlal v17.8h, v4.8b, v1.8b
prfm pldl1strm, [x1]
.ifc \codec,h264
rshrn v16.8b, v16.8h, #6
rshrn v17.8b, v17.8h, #6
.else
add v16.8h, v16.8h, v22.8h
add v17.8h, v17.8h, v22.8h
shrn v16.8b, v16.8h, #6
shrn v17.8b, v17.8h, #6
.endif
prfm pldl1strm, [x1, x2]
.ifc \type,avg
ld1 {v20.8b}, [x8], x2
ld1 {v21.8b}, [x8], x2
urhadd v16.8b, v16.8b, v20.8b
urhadd v17.8b, v17.8b, v21.8b
.endif
subs w3, w3, #2
st1 {v16.8b}, [x0], x2
st1 {v17.8b}, [x0], x2
b.gt 3b
ret
4: ld1 {v4.8b, v5.8b}, [x1], x2
ld1 {v6.8b, v7.8b}, [x1], x2
ext v5.8b, v4.8b, v5.8b, #1
ext v7.8b, v6.8b, v7.8b, #1
prfm pldl1strm, [x1]
subs w3, w3, #2
umull v16.8h, v4.8b, v0.8b
umlal v16.8h, v5.8b, v1.8b
umull v17.8h, v6.8b, v0.8b
umlal v17.8h, v7.8b, v1.8b
prfm pldl1strm, [x1, x2]
.ifc \codec,h264
rshrn v16.8b, v16.8h, #6
rshrn v17.8b, v17.8h, #6
.else
add v16.8h, v16.8h, v22.8h
add v17.8h, v17.8h, v22.8h
shrn v16.8b, v16.8h, #6
shrn v17.8b, v17.8h, #6
.endif
.ifc \type,avg
ld1 {v20.8b}, [x8], x2
ld1 {v21.8b}, [x8], x2
urhadd v16.8b, v16.8b, v20.8b
urhadd v17.8b, v17.8b, v21.8b
.endif
st1 {v16.8b}, [x0], x2
st1 {v17.8b}, [x0], x2
b.gt 4b
ret
5: ld1 {v4.8b}, [x1], x2
ld1 {v5.8b}, [x1], x2
prfm pldl1strm, [x1]
subs w3, w3, #2
umull v16.8h, v4.8b, v0.8b
umull v17.8h, v5.8b, v0.8b
prfm pldl1strm, [x1, x2]
.ifc \codec,h264
rshrn v16.8b, v16.8h, #6
rshrn v17.8b, v17.8h, #6
.else
add v16.8h, v16.8h, v22.8h
add v17.8h, v17.8h, v22.8h
shrn v16.8b, v16.8h, #6
shrn v17.8b, v17.8h, #6
.endif
.ifc \type,avg
ld1 {v20.8b}, [x8], x2
ld1 {v21.8b}, [x8], x2
urhadd v16.8b, v16.8b, v20.8b
urhadd v17.8b, v17.8b, v21.8b
.endif
st1 {v16.8b}, [x0], x2
st1 {v17.8b}, [x0], x2
b.gt 5b
ret
endfunc
.endm
/* chroma_mc4(uint8_t *dst, const uint8_t *src, ptrdiff_t stride, int h, int x, int y) */
.macro h264_chroma_mc4 type, codec=h264
function ff_\type\()_\codec\()_chroma_mc4_neon, export=1
.ifc \type,avg
mov x8, x0
.endif
prfm pldl1strm, [x1]
prfm pldl1strm, [x1, x2]
.ifc \codec,rv40
movrel x6, rv40bias
lsr w9, w5, #1
lsr w10, w4, #1
lsl w9, w9, #3
lsl w10, w10, #1
add w9, w9, w10
add x6, x6, w9, uxtw
ld1r {v22.8h}, [x6]
.endif
.ifc \codec,vc1
movi v22.8h, #28
.endif
mul w7, w4, w5
lsl w14, w5, #3
lsl w13, w4, #3
cmp w7, #0
sub w6, w14, w7
sub w12, w13, w7
sub w4, w7, w13
sub w4, w4, w14
add w4, w4, #64
b.eq 2f
dup v24.8b, w4
dup v25.8b, w12
ld1 {v4.8b}, [x1], x2
dup v26.8b, w6
dup v27.8b, w7
ext v5.8b, v4.8b, v5.8b, #1
trn1 v0.2s, v24.2s, v25.2s
trn1 v2.2s, v26.2s, v27.2s
trn1 v4.2s, v4.2s, v5.2s
1: ld1 {v6.8b}, [x1], x2
ext v7.8b, v6.8b, v7.8b, #1
trn1 v6.2s, v6.2s, v7.2s
umull v18.8h, v4.8b, v0.8b
umlal v18.8h, v6.8b, v2.8b
ld1 {v4.8b}, [x1], x2
ext v5.8b, v4.8b, v5.8b, #1
trn1 v4.2s, v4.2s, v5.2s
prfm pldl1strm, [x1]
umull v19.8h, v6.8b, v0.8b
umlal v19.8h, v4.8b, v2.8b
trn1 v30.2d, v18.2d, v19.2d
trn2 v31.2d, v18.2d, v19.2d
add v18.8h, v30.8h, v31.8h
.ifc \codec,h264
rshrn v16.8b, v18.8h, #6
.else
add v18.8h, v18.8h, v22.8h
shrn v16.8b, v18.8h, #6
.endif
subs w3, w3, #2
prfm pldl1strm, [x1, x2]
.ifc \type,avg
ld1 {v20.s}[0], [x8], x2
ld1 {v20.s}[1], [x8], x2
urhadd v16.8b, v16.8b, v20.8b
.endif
st1 {v16.s}[0], [x0], x2
st1 {v16.s}[1], [x0], x2
b.gt 1b
ret
2: adds w12, w12, w6
dup v30.8b, w4
b.eq 5f
tst w6, w6
dup v31.8b, w12
trn1 v0.2s, v30.2s, v31.2s
trn2 v1.2s, v30.2s, v31.2s
b.eq 4f
ext v1.8b, v0.8b, v1.8b, #4
ld1 {v4.s}[0], [x1], x2
3: ld1 {v4.s}[1], [x1], x2
umull v18.8h, v4.8b, v0.8b
ld1 {v4.s}[0], [x1], x2
umull v19.8h, v4.8b, v1.8b
trn1 v30.2d, v18.2d, v19.2d
trn2 v31.2d, v18.2d, v19.2d
add v18.8h, v30.8h, v31.8h
prfm pldl1strm, [x1]
.ifc \codec,h264
rshrn v16.8b, v18.8h, #6
.else
add v18.8h, v18.8h, v22.8h
shrn v16.8b, v18.8h, #6
.endif
.ifc \type,avg
ld1 {v20.s}[0], [x8], x2
ld1 {v20.s}[1], [x8], x2
urhadd v16.8b, v16.8b, v20.8b
.endif
subs w3, w3, #2
prfm pldl1strm, [x1, x2]
st1 {v16.s}[0], [x0], x2
st1 {v16.s}[1], [x0], x2
b.gt 3b
ret
4: ld1 {v4.8b}, [x1], x2
ld1 {v6.8b}, [x1], x2
ext v5.8b, v4.8b, v5.8b, #1
ext v7.8b, v6.8b, v7.8b, #1
trn1 v4.2s, v4.2s, v5.2s
trn1 v6.2s, v6.2s, v7.2s
umull v18.8h, v4.8b, v0.8b
umull v19.8h, v6.8b, v0.8b
subs w3, w3, #2
trn1 v30.2d, v18.2d, v19.2d
trn2 v31.2d, v18.2d, v19.2d
add v18.8h, v30.8h, v31.8h
prfm pldl1strm, [x1]
.ifc \codec,h264
rshrn v16.8b, v18.8h, #6
.else
add v18.8h, v18.8h, v22.8h
shrn v16.8b, v18.8h, #6
.endif
.ifc \type,avg
ld1 {v20.s}[0], [x8], x2
ld1 {v20.s}[1], [x8], x2
urhadd v16.8b, v16.8b, v20.8b
.endif
prfm pldl1strm, [x1]
st1 {v16.s}[0], [x0], x2
st1 {v16.s}[1], [x0], x2
b.gt 4b
ret
5: ld1 {v4.s}[0], [x1], x2
ld1 {v4.s}[1], [x1], x2
umull v18.8h, v4.8b, v30.8b
subs w3, w3, #2
prfm pldl1strm, [x1]
.ifc \codec,h264
rshrn v16.8b, v18.8h, #6
.else
add v18.8h, v18.8h, v22.8h
shrn v16.8b, v18.8h, #6
.endif
.ifc \type,avg
ld1 {v20.s}[0], [x8], x2
ld1 {v20.s}[1], [x8], x2
urhadd v16.8b, v16.8b, v20.8b
.endif
prfm pldl1strm, [x1]
st1 {v16.s}[0], [x0], x2
st1 {v16.s}[1], [x0], x2
b.gt 5b
ret
endfunc
.endm
.macro h264_chroma_mc2 type
function ff_\type\()_h264_chroma_mc2_neon, export=1
prfm pldl1strm, [x1]
prfm pldl1strm, [x1, x2]
orr w7, w4, w5
cbz w7, 2f
mul w7, w4, w5
lsl w14, w5, #3
lsl w13, w4, #3
sub w6, w14, w7
sub w12, w13, w7
sub w4, w7, w13
sub w4, w4, w14
add w4, w4, #64
dup v0.8b, w4
dup v2.8b, w12
dup v1.8b, w6
dup v3.8b, w7
trn1 v0.4h, v0.4h, v2.4h
trn1 v1.4h, v1.4h, v3.4h
1:
ld1 {v4.s}[0], [x1], x2
ld1 {v4.s}[1], [x1], x2
rev64 v5.2s, v4.2s
ld1 {v5.s}[1], [x1]
ext v6.8b, v4.8b, v5.8b, #1
ext v7.8b, v5.8b, v4.8b, #1
trn1 v4.4h, v4.4h, v6.4h
trn1 v5.4h, v5.4h, v7.4h
umull v16.8h, v4.8b, v0.8b
umlal v16.8h, v5.8b, v1.8b
.ifc \type,avg
ld1 {v18.h}[0], [x0], x2
ld1 {v18.h}[2], [x0]
sub x0, x0, x2
.endif
rev64 v17.4s, v16.4s
add v16.8h, v16.8h, v17.8h
rshrn v16.8b, v16.8h, #6
.ifc \type,avg
urhadd v16.8b, v16.8b, v18.8b
.endif
st1 {v16.h}[0], [x0], x2
st1 {v16.h}[2], [x0], x2
subs w3, w3, #2
b.gt 1b
ret
2:
ld1 {v16.h}[0], [x1], x2
ld1 {v16.h}[1], [x1], x2
.ifc \type,avg
ld1 {v18.h}[0], [x0], x2
ld1 {v18.h}[1], [x0]
sub x0, x0, x2
urhadd v16.8b, v16.8b, v18.8b
.endif
st1 {v16.h}[0], [x0], x2
st1 {v16.h}[1], [x0], x2
subs w3, w3, #2
b.gt 2b
ret
endfunc
.endm
h264_chroma_mc8 put
h264_chroma_mc8 avg
h264_chroma_mc4 put
h264_chroma_mc4 avg
h264_chroma_mc2 put
h264_chroma_mc2 avg
#if CONFIG_RV40_DECODER
const rv40bias
.short 0, 16, 32, 16
.short 32, 28, 32, 28
.short 0, 32, 16, 32
.short 32, 28, 32, 28
endconst
h264_chroma_mc8 put, rv40
h264_chroma_mc8 avg, rv40
h264_chroma_mc4 put, rv40
h264_chroma_mc4 avg, rv40
#endif
#if CONFIG_VC1DSP
h264_chroma_mc8 put, vc1
h264_chroma_mc8 avg, vc1
h264_chroma_mc4 put, vc1
h264_chroma_mc4 avg, vc1
#endif
@@ -0,0 +1,166 @@
/*
* Copyright (c) 2010 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/h264dsp.h"
void ff_h264_v_loop_filter_luma_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_h_loop_filter_luma_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_v_loop_filter_luma_intra_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta);
void ff_h264_h_loop_filter_luma_intra_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta);
void ff_h264_v_loop_filter_chroma_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_h_loop_filter_chroma_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_h_loop_filter_chroma422_neon(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_v_loop_filter_chroma_intra_neon(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_h264_h_loop_filter_chroma_intra_neon(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_h264_h_loop_filter_chroma422_intra_neon(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_h264_h_loop_filter_chroma_mbaff_intra_neon(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_weight_h264_pixels_16_neon(uint8_t *dst, ptrdiff_t stride, int height,
int log2_den, int weight, int offset);
void ff_weight_h264_pixels_8_neon(uint8_t *dst, ptrdiff_t stride, int height,
int log2_den, int weight, int offset);
void ff_weight_h264_pixels_4_neon(uint8_t *dst, ptrdiff_t stride, int height,
int log2_den, int weight, int offset);
void ff_biweight_h264_pixels_16_neon(uint8_t *dst, uint8_t *src, ptrdiff_t stride,
int height, int log2_den, int weightd,
int weights, int offset);
void ff_biweight_h264_pixels_8_neon(uint8_t *dst, uint8_t *src, ptrdiff_t stride,
int height, int log2_den, int weightd,
int weights, int offset);
void ff_biweight_h264_pixels_4_neon(uint8_t *dst, uint8_t *src, ptrdiff_t stride,
int height, int log2_den, int weightd,
int weights, int offset);
void ff_h264_idct_add_neon(uint8_t *dst, int16_t *block, int stride);
void ff_h264_idct_dc_add_neon(uint8_t *dst, int16_t *block, int stride);
void ff_h264_idct_add16_neon(uint8_t *dst, const int *block_offset,
int16_t *block, int stride,
const uint8_t nnzc[5 * 8]);
void ff_h264_idct_add16intra_neon(uint8_t *dst, const int *block_offset,
int16_t *block, int stride,
const uint8_t nnzc[5 * 8]);
void ff_h264_idct_add8_neon(uint8_t **dest, const int *block_offset,
int16_t *block, int stride,
const uint8_t nnzc[15 * 8]);
void ff_h264_idct8_add_neon(uint8_t *dst, int16_t *block, int stride);
void ff_h264_idct8_dc_add_neon(uint8_t *dst, int16_t *block, int stride);
void ff_h264_idct8_add4_neon(uint8_t *dst, const int *block_offset,
int16_t *block, int stride,
const uint8_t nnzc[5 * 8]);
void ff_h264_v_loop_filter_luma_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_h_loop_filter_luma_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_v_loop_filter_luma_intra_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta);
void ff_h264_h_loop_filter_luma_intra_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta);
void ff_h264_v_loop_filter_chroma_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_h_loop_filter_chroma_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_h_loop_filter_chroma422_neon_10(uint8_t *pix, ptrdiff_t stride, int alpha,
int beta, int8_t *tc0);
void ff_h264_v_loop_filter_chroma_intra_neon_10(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_h264_h_loop_filter_chroma_intra_neon_10(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_h264_h_loop_filter_chroma422_intra_neon_10(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
void ff_h264_h_loop_filter_chroma_mbaff_intra_neon_10(uint8_t *pix, ptrdiff_t stride,
int alpha, int beta);
av_cold void ff_h264dsp_init_aarch64(H264DSPContext *c, const int bit_depth,
const int chroma_format_idc)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags) && bit_depth == 8) {
c->h264_v_loop_filter_luma = ff_h264_v_loop_filter_luma_neon;
c->h264_h_loop_filter_luma = ff_h264_h_loop_filter_luma_neon;
c->h264_v_loop_filter_luma_intra= ff_h264_v_loop_filter_luma_intra_neon;
c->h264_h_loop_filter_luma_intra= ff_h264_h_loop_filter_luma_intra_neon;
c->h264_v_loop_filter_chroma = ff_h264_v_loop_filter_chroma_neon;
c->h264_v_loop_filter_chroma_intra = ff_h264_v_loop_filter_chroma_intra_neon;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_h264_h_loop_filter_chroma_neon;
c->h264_h_loop_filter_chroma_intra = ff_h264_h_loop_filter_chroma_intra_neon;
c->h264_h_loop_filter_chroma_mbaff_intra = ff_h264_h_loop_filter_chroma_mbaff_intra_neon;
} else {
c->h264_h_loop_filter_chroma = ff_h264_h_loop_filter_chroma422_neon;
c->h264_h_loop_filter_chroma_mbaff = ff_h264_h_loop_filter_chroma_neon;
c->h264_h_loop_filter_chroma_intra = ff_h264_h_loop_filter_chroma422_intra_neon;
c->h264_h_loop_filter_chroma_mbaff_intra = ff_h264_h_loop_filter_chroma_intra_neon;
}
c->weight_h264_pixels_tab[0] = ff_weight_h264_pixels_16_neon;
c->weight_h264_pixels_tab[1] = ff_weight_h264_pixels_8_neon;
c->weight_h264_pixels_tab[2] = ff_weight_h264_pixels_4_neon;
c->biweight_h264_pixels_tab[0] = ff_biweight_h264_pixels_16_neon;
c->biweight_h264_pixels_tab[1] = ff_biweight_h264_pixels_8_neon;
c->biweight_h264_pixels_tab[2] = ff_biweight_h264_pixels_4_neon;
c->h264_idct_add = ff_h264_idct_add_neon;
c->h264_idct_dc_add = ff_h264_idct_dc_add_neon;
c->h264_idct_add16 = ff_h264_idct_add16_neon;
c->h264_idct_add16intra = ff_h264_idct_add16intra_neon;
if (chroma_format_idc <= 1)
c->h264_idct_add8 = ff_h264_idct_add8_neon;
c->h264_idct8_add = ff_h264_idct8_add_neon;
c->h264_idct8_dc_add = ff_h264_idct8_dc_add_neon;
c->h264_idct8_add4 = ff_h264_idct8_add4_neon;
} else if (have_neon(cpu_flags) && bit_depth == 10) {
c->h264_v_loop_filter_chroma = ff_h264_v_loop_filter_chroma_neon_10;
c->h264_v_loop_filter_chroma_intra = ff_h264_v_loop_filter_chroma_intra_neon_10;
if (chroma_format_idc <= 1) {
c->h264_h_loop_filter_chroma = ff_h264_h_loop_filter_chroma_neon_10;
c->h264_h_loop_filter_chroma_intra = ff_h264_h_loop_filter_chroma_intra_neon_10;
c->h264_h_loop_filter_chroma_mbaff_intra = ff_h264_h_loop_filter_chroma_mbaff_intra_neon_10;
} else {
c->h264_h_loop_filter_chroma = ff_h264_h_loop_filter_chroma422_neon_10;
c->h264_h_loop_filter_chroma_mbaff = ff_h264_h_loop_filter_chroma_neon_10;
c->h264_h_loop_filter_chroma_intra = ff_h264_h_loop_filter_chroma422_intra_neon_10;
c->h264_h_loop_filter_chroma_mbaff_intra = ff_h264_h_loop_filter_chroma_intra_neon_10;
}
}
}
File diff suppressed because it is too large. Load diff
+415
View File
@@ -0,0 +1,415 @@
/*
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2013 Janne Grunau <janne-libav@jannau.net>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
function ff_h264_idct_add_neon, export=1
.L_ff_h264_idct_add_neon:
AARCH64_VALID_CALL_TARGET
ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x1]
sxtw x2, w2
movi v30.8h, #0
add v4.4h, v0.4h, v2.4h
sshr v16.4h, v1.4h, #1
st1 {v30.8h}, [x1], #16
sshr v17.4h, v3.4h, #1
st1 {v30.8h}, [x1], #16
sub v5.4h, v0.4h, v2.4h
sub v6.4h, v16.4h, v3.4h
add v7.4h, v1.4h, v17.4h
add v0.4h, v4.4h, v7.4h
add v1.4h, v5.4h, v6.4h
sub v2.4h, v5.4h, v6.4h
sub v3.4h, v4.4h, v7.4h
transpose_4x4H v0, v1, v2, v3, v4, v5, v6, v7
add v4.4h, v0.4h, v2.4h
ld1 {v18.s}[0], [x0], x2
sshr v16.4h, v3.4h, #1
sshr v17.4h, v1.4h, #1
ld1 {v18.s}[1], [x0], x2
sub v5.4h, v0.4h, v2.4h
ld1 {v19.s}[1], [x0], x2
add v6.4h, v16.4h, v1.4h
ins v4.d[1], v5.d[0]
sub v7.4h, v17.4h, v3.4h
ld1 {v19.s}[0], [x0], x2
ins v6.d[1], v7.d[0]
sub x0, x0, x2, lsl #2
add v0.8h, v4.8h, v6.8h
sub v1.8h, v4.8h, v6.8h
srshr v0.8h, v0.8h, #6
srshr v1.8h, v1.8h, #6
uaddw v0.8h, v0.8h, v18.8b
uaddw v1.8h, v1.8h, v19.8b
sqxtun v0.8b, v0.8h
sqxtun v1.8b, v1.8h
st1 {v0.s}[0], [x0], x2
st1 {v0.s}[1], [x0], x2
st1 {v1.s}[1], [x0], x2
st1 {v1.s}[0], [x0], x2
sub x1, x1, #32
ret
endfunc
function ff_h264_idct_dc_add_neon, export=1
.L_ff_h264_idct_dc_add_neon:
AARCH64_VALID_CALL_TARGET
sxtw x2, w2
mov w3, #0
ld1r {v2.8h}, [x1]
strh w3, [x1]
srshr v2.8h, v2.8h, #6
ld1 {v0.s}[0], [x0], x2
ld1 {v0.s}[1], [x0], x2
uaddw v3.8h, v2.8h, v0.8b
ld1 {v1.s}[0], [x0], x2
ld1 {v1.s}[1], [x0], x2
uaddw v4.8h, v2.8h, v1.8b
sqxtun v0.8b, v3.8h
sqxtun v1.8b, v4.8h
sub x0, x0, x2, lsl #2
st1 {v0.s}[0], [x0], x2
st1 {v0.s}[1], [x0], x2
st1 {v1.s}[0], [x0], x2
st1 {v1.s}[1], [x0], x2
ret
endfunc
function ff_h264_idct_add16_neon, export=1
mov x12, x30
mov x6, x0 // dest
mov x5, x1 // block_offset
mov x1, x2 // block
mov w9, w3 // stride
movrel x7, scan8
mov x10, #16
movrel x13, .L_ff_h264_idct_dc_add_neon
movrel x14, .L_ff_h264_idct_add_neon
1: mov w2, w9
ldrb w3, [x7], #1
ldrsw x0, [x5], #4
ldrb w3, [x4, w3, uxtw]
subs w3, w3, #1
b.lt 2f
ldrsh w3, [x1]
add x0, x0, x6
ccmp w3, #0, #4, eq
csel x15, x13, x14, ne
blr x15
2: subs x10, x10, #1
add x1, x1, #32
b.ne 1b
ret x12
endfunc
function ff_h264_idct_add16intra_neon, export=1
mov x12, x30
mov x6, x0 // dest
mov x5, x1 // block_offset
mov x1, x2 // block
mov w9, w3 // stride
movrel x7, scan8
mov x10, #16
movrel x13, .L_ff_h264_idct_dc_add_neon
movrel x14, .L_ff_h264_idct_add_neon
1: mov w2, w9
ldrb w3, [x7], #1
ldrsw x0, [x5], #4
ldrb w3, [x4, w3, uxtw]
add x0, x0, x6
cmp w3, #0
ldrsh w3, [x1]
csel x15, x13, x14, eq
ccmp w3, #0, #0, eq
b.eq 2f
blr x15
2: subs x10, x10, #1
add x1, x1, #32
b.ne 1b
ret x12
endfunc
function ff_h264_idct_add8_neon, export=1
stp x19, x20, [sp, #-0x40]!
mov x12, x30
ldp x6, x15, [x0] // dest[0], dest[1]
add x5, x1, #16*4 // block_offset
add x9, x2, #16*32 // block
mov w19, w3 // stride
movrel x13, .L_ff_h264_idct_dc_add_neon
movrel x14, .L_ff_h264_idct_add_neon
movrel x7, scan8, 16
mov x10, #0
mov x11, #16
1: mov w2, w19
ldrb w3, [x7, x10] // scan8[i]
ldrsw x0, [x5, x10, lsl #2] // block_offset[i]
ldrb w3, [x4, w3, uxtw] // nnzc[ scan8[i] ]
add x0, x0, x6 // block_offset[i] + dst[j-1]
add x1, x9, x10, lsl #5 // block + i * 16
cmp w3, #0
ldrsh w3, [x1] // block[i*16]
csel x20, x13, x14, eq
ccmp w3, #0, #0, eq
b.eq 2f
blr x20
2: add x10, x10, #1
cmp x10, #4
csel x10, x11, x10, eq // mov x10, #16
csel x6, x15, x6, eq
cmp x10, #20
b.lt 1b
ldp x19, x20, [sp], #0x40
ret x12
endfunc
.macro idct8x8_cols pass
.if \pass == 0
va .req v18
vb .req v30
sshr v18.8h, v26.8h, #1
add v16.8h, v24.8h, v28.8h
ld1 {v30.8h, v31.8h}, [x1]
st1 {v19.8h}, [x1], #16
st1 {v19.8h}, [x1], #16
sub v17.8h, v24.8h, v28.8h
sshr v19.8h, v30.8h, #1
sub v18.8h, v18.8h, v30.8h
add v19.8h, v19.8h, v26.8h
.else
va .req v30
vb .req v18
sshr v30.8h, v26.8h, #1
sshr v19.8h, v18.8h, #1
add v16.8h, v24.8h, v28.8h
sub v17.8h, v24.8h, v28.8h
sub v30.8h, v30.8h, v18.8h
add v19.8h, v19.8h, v26.8h
.endif
add v26.8h, v17.8h, va.8h
sub v28.8h, v17.8h, va.8h
add v24.8h, v16.8h, v19.8h
sub vb.8h, v16.8h, v19.8h
sub v16.8h, v29.8h, v27.8h
add v17.8h, v31.8h, v25.8h
sub va.8h, v31.8h, v25.8h
add v19.8h, v29.8h, v27.8h
sub v16.8h, v16.8h, v31.8h
sub v17.8h, v17.8h, v27.8h
add va.8h, va.8h, v29.8h
add v19.8h, v19.8h, v25.8h
sshr v25.8h, v25.8h, #1
sshr v27.8h, v27.8h, #1
sshr v29.8h, v29.8h, #1
sshr v31.8h, v31.8h, #1
sub v16.8h, v16.8h, v31.8h
sub v17.8h, v17.8h, v27.8h
add va.8h, va.8h, v29.8h
add v19.8h, v19.8h, v25.8h
sshr v25.8h, v16.8h, #2
sshr v27.8h, v17.8h, #2
sshr v29.8h, va.8h, #2
sshr v31.8h, v19.8h, #2
sub v19.8h, v19.8h, v25.8h
sub va.8h, v27.8h, va.8h
add v17.8h, v17.8h, v29.8h
add v16.8h, v16.8h, v31.8h
.if \pass == 0
sub v31.8h, v24.8h, v19.8h
add v24.8h, v24.8h, v19.8h
add v25.8h, v26.8h, v18.8h
sub v18.8h, v26.8h, v18.8h
add v26.8h, v28.8h, v17.8h
add v27.8h, v30.8h, v16.8h
sub v29.8h, v28.8h, v17.8h
sub v28.8h, v30.8h, v16.8h
.else
sub v31.8h, v24.8h, v19.8h
add v24.8h, v24.8h, v19.8h
add v25.8h, v26.8h, v30.8h
sub v30.8h, v26.8h, v30.8h
add v26.8h, v28.8h, v17.8h
sub v29.8h, v28.8h, v17.8h
add v27.8h, v18.8h, v16.8h
sub v28.8h, v18.8h, v16.8h
.endif
.unreq va
.unreq vb
.endm
function ff_h264_idct8_add_neon, export=1
.L_ff_h264_idct8_add_neon:
AARCH64_VALID_CALL_TARGET
movi v19.8h, #0
sxtw x2, w2
ld1 {v24.8h, v25.8h}, [x1]
st1 {v19.8h}, [x1], #16
st1 {v19.8h}, [x1], #16
ld1 {v26.8h, v27.8h}, [x1]
st1 {v19.8h}, [x1], #16
st1 {v19.8h}, [x1], #16
ld1 {v28.8h, v29.8h}, [x1]
st1 {v19.8h}, [x1], #16
st1 {v19.8h}, [x1], #16
idct8x8_cols 0
transpose_8x8H v24, v25, v26, v27, v28, v29, v18, v31, v6, v7
idct8x8_cols 1
mov x3, x0
srshr v24.8h, v24.8h, #6
ld1 {v0.8b}, [x0], x2
srshr v25.8h, v25.8h, #6
ld1 {v1.8b}, [x0], x2
srshr v26.8h, v26.8h, #6
ld1 {v2.8b}, [x0], x2
srshr v27.8h, v27.8h, #6
ld1 {v3.8b}, [x0], x2
srshr v28.8h, v28.8h, #6
ld1 {v4.8b}, [x0], x2
srshr v29.8h, v29.8h, #6
ld1 {v5.8b}, [x0], x2
srshr v30.8h, v30.8h, #6
ld1 {v6.8b}, [x0], x2
srshr v31.8h, v31.8h, #6
ld1 {v7.8b}, [x0], x2
uaddw v24.8h, v24.8h, v0.8b
uaddw v25.8h, v25.8h, v1.8b
uaddw v26.8h, v26.8h, v2.8b
sqxtun v0.8b, v24.8h
uaddw v27.8h, v27.8h, v3.8b
sqxtun v1.8b, v25.8h
uaddw v28.8h, v28.8h, v4.8b
sqxtun v2.8b, v26.8h
st1 {v0.8b}, [x3], x2
uaddw v29.8h, v29.8h, v5.8b
sqxtun v3.8b, v27.8h
st1 {v1.8b}, [x3], x2
uaddw v30.8h, v30.8h, v6.8b
sqxtun v4.8b, v28.8h
st1 {v2.8b}, [x3], x2
uaddw v31.8h, v31.8h, v7.8b
sqxtun v5.8b, v29.8h
st1 {v3.8b}, [x3], x2
sqxtun v6.8b, v30.8h
sqxtun v7.8b, v31.8h
st1 {v4.8b}, [x3], x2
st1 {v5.8b}, [x3], x2
st1 {v6.8b}, [x3], x2
st1 {v7.8b}, [x3], x2
sub x1, x1, #128
ret
endfunc
function ff_h264_idct8_dc_add_neon, export=1
.L_ff_h264_idct8_dc_add_neon:
AARCH64_VALID_CALL_TARGET
mov w3, #0
sxtw x2, w2
ld1r {v31.8h}, [x1]
strh w3, [x1]
ld1 {v0.8b}, [x0], x2
srshr v31.8h, v31.8h, #6
ld1 {v1.8b}, [x0], x2
ld1 {v2.8b}, [x0], x2
uaddw v24.8h, v31.8h, v0.8b
ld1 {v3.8b}, [x0], x2
uaddw v25.8h, v31.8h, v1.8b
ld1 {v4.8b}, [x0], x2
uaddw v26.8h, v31.8h, v2.8b
ld1 {v5.8b}, [x0], x2
uaddw v27.8h, v31.8h, v3.8b
ld1 {v6.8b}, [x0], x2
uaddw v28.8h, v31.8h, v4.8b
ld1 {v7.8b}, [x0], x2
uaddw v29.8h, v31.8h, v5.8b
uaddw v30.8h, v31.8h, v6.8b
uaddw v31.8h, v31.8h, v7.8b
sqxtun v0.8b, v24.8h
sqxtun v1.8b, v25.8h
sqxtun v2.8b, v26.8h
sqxtun v3.8b, v27.8h
sub x0, x0, x2, lsl #3
st1 {v0.8b}, [x0], x2
sqxtun v4.8b, v28.8h
st1 {v1.8b}, [x0], x2
sqxtun v5.8b, v29.8h
st1 {v2.8b}, [x0], x2
sqxtun v6.8b, v30.8h
st1 {v3.8b}, [x0], x2
sqxtun v7.8b, v31.8h
st1 {v4.8b}, [x0], x2
st1 {v5.8b}, [x0], x2
st1 {v6.8b}, [x0], x2
st1 {v7.8b}, [x0], x2
ret
endfunc
function ff_h264_idct8_add4_neon, export=1
mov x12, x30
mov x6, x0
mov x5, x1
mov x1, x2
mov w2, w3
movrel x7, scan8
mov w10, #16
movrel x13, .L_ff_h264_idct8_dc_add_neon
movrel x14, .L_ff_h264_idct8_add_neon
1: ldrb w9, [x7], #4
ldrsw x0, [x5], #16
ldrb w9, [x4, w9, uxtw]
subs w9, w9, #1
b.lt 2f
ldrsh w11, [x1]
add x0, x6, x0
ccmp w11, #0, #4, eq
csel x15, x13, x14, ne
blr x15
2: subs w10, w10, #4
add x1, x1, #128
b.ne 1b
ret x12
endfunc
const scan8
.byte 4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8
.byte 6+ 1*8, 7+ 1*8, 6+ 2*8, 7+ 2*8
.byte 4+ 3*8, 5+ 3*8, 4+ 4*8, 5+ 4*8
.byte 6+ 3*8, 7+ 3*8, 6+ 4*8, 7+ 4*8
.byte 4+ 6*8, 5+ 6*8, 4+ 7*8, 5+ 7*8
.byte 6+ 6*8, 7+ 6*8, 6+ 7*8, 7+ 7*8
.byte 4+ 8*8, 5+ 8*8, 4+ 9*8, 5+ 9*8
.byte 6+ 8*8, 7+ 8*8, 6+ 9*8, 7+ 9*8
.byte 4+11*8, 5+11*8, 4+12*8, 5+12*8
.byte 6+11*8, 7+11*8, 6+12*8, 7+12*8
.byte 4+13*8, 5+13*8, 4+14*8, 5+14*8
.byte 6+13*8, 7+13*8, 6+14*8, 7+14*8
endconst
+135
View File
@@ -0,0 +1,135 @@
/*
* Copyright (c) 2009 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/h264pred.h"
void ff_pred16x16_vert_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_hor_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_plane_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_128_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_left_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_top_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_vert_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_hor_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_plane_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_128_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_left_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_top_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_l0t_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_0lt_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_l00_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_0l0_dc_neon(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_vert_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_hor_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_plane_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred16x16_top_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_vert_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_hor_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_plane_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_128_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_left_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_top_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_l0t_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_0lt_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_l00_dc_neon_10(uint8_t *src, ptrdiff_t stride);
void ff_pred8x8_0l0_dc_neon_10(uint8_t *src, ptrdiff_t stride);
static av_cold void h264_pred_init_neon(H264PredContext *h, int codec_id,
const int bit_depth,
const int chroma_format_idc)
{
if (bit_depth == 8) {
if (chroma_format_idc <= 1) {
h->pred8x8[VERT_PRED8x8 ] = ff_pred8x8_vert_neon;
h->pred8x8[HOR_PRED8x8 ] = ff_pred8x8_hor_neon;
if (codec_id != AV_CODEC_ID_VP7 && codec_id != AV_CODEC_ID_VP8)
h->pred8x8[PLANE_PRED8x8] = ff_pred8x8_plane_neon;
h->pred8x8[DC_128_PRED8x8 ] = ff_pred8x8_128_dc_neon;
if (codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP7 &&
codec_id != AV_CODEC_ID_VP8) {
h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_neon;
h->pred8x8[LEFT_DC_PRED8x8] = ff_pred8x8_left_dc_neon;
h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_neon;
h->pred8x8[ALZHEIMER_DC_L0T_PRED8x8] = ff_pred8x8_l0t_dc_neon;
h->pred8x8[ALZHEIMER_DC_0LT_PRED8x8] = ff_pred8x8_0lt_dc_neon;
h->pred8x8[ALZHEIMER_DC_L00_PRED8x8] = ff_pred8x8_l00_dc_neon;
h->pred8x8[ALZHEIMER_DC_0L0_PRED8x8] = ff_pred8x8_0l0_dc_neon;
}
}
h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_neon;
h->pred16x16[VERT_PRED8x8 ] = ff_pred16x16_vert_neon;
h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_hor_neon;
h->pred16x16[LEFT_DC_PRED8x8] = ff_pred16x16_left_dc_neon;
h->pred16x16[TOP_DC_PRED8x8 ] = ff_pred16x16_top_dc_neon;
h->pred16x16[DC_128_PRED8x8 ] = ff_pred16x16_128_dc_neon;
if (codec_id != AV_CODEC_ID_SVQ3 && codec_id != AV_CODEC_ID_RV40 &&
codec_id != AV_CODEC_ID_VP7 && codec_id != AV_CODEC_ID_VP8)
h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_plane_neon;
}
if (bit_depth == 10) {
if (chroma_format_idc <= 1) {
h->pred8x8[VERT_PRED8x8 ] = ff_pred8x8_vert_neon_10;
h->pred8x8[HOR_PRED8x8 ] = ff_pred8x8_hor_neon_10;
if (codec_id != AV_CODEC_ID_VP7 && codec_id != AV_CODEC_ID_VP8)
h->pred8x8[PLANE_PRED8x8] = ff_pred8x8_plane_neon_10;
h->pred8x8[DC_128_PRED8x8 ] = ff_pred8x8_128_dc_neon_10;
if (codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP7 &&
codec_id != AV_CODEC_ID_VP8) {
h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_neon_10;
h->pred8x8[LEFT_DC_PRED8x8] = ff_pred8x8_left_dc_neon_10;
h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_neon_10;
h->pred8x8[ALZHEIMER_DC_L0T_PRED8x8] = ff_pred8x8_l0t_dc_neon_10;
h->pred8x8[ALZHEIMER_DC_0LT_PRED8x8] = ff_pred8x8_0lt_dc_neon_10;
h->pred8x8[ALZHEIMER_DC_L00_PRED8x8] = ff_pred8x8_l00_dc_neon_10;
h->pred8x8[ALZHEIMER_DC_0L0_PRED8x8] = ff_pred8x8_0l0_dc_neon_10;
}
}
h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_neon_10;
h->pred16x16[VERT_PRED8x8 ] = ff_pred16x16_vert_neon_10;
h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_hor_neon_10;
h->pred16x16[TOP_DC_PRED8x8 ] = ff_pred16x16_top_dc_neon_10;
if (codec_id != AV_CODEC_ID_SVQ3 && codec_id != AV_CODEC_ID_RV40 &&
codec_id != AV_CODEC_ID_VP7 && codec_id != AV_CODEC_ID_VP8)
h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_plane_neon_10;
}
}
av_cold void ff_h264_pred_init_aarch64(H264PredContext *h, int codec_id,
int bit_depth, const int chroma_format_idc)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags))
h264_pred_init_neon(h, codec_id, bit_depth, chroma_format_idc);
}
+762
View File
@@ -0,0 +1,762 @@
/*
* Copyright (c) 2009 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
.macro ldcol.8 rd, rs, rt, n=8, hi=0
.if \n >= 8 || \hi == 0
ld1 {\rd\().b}[0], [\rs], \rt
ld1 {\rd\().b}[1], [\rs], \rt
ld1 {\rd\().b}[2], [\rs], \rt
ld1 {\rd\().b}[3], [\rs], \rt
.endif
.if \n >= 8 || \hi == 1
ld1 {\rd\().b}[4], [\rs], \rt
ld1 {\rd\().b}[5], [\rs], \rt
ld1 {\rd\().b}[6], [\rs], \rt
ld1 {\rd\().b}[7], [\rs], \rt
.endif
.if \n == 16
ld1 {\rd\().b}[8], [\rs], \rt
ld1 {\rd\().b}[9], [\rs], \rt
ld1 {\rd\().b}[10], [\rs], \rt
ld1 {\rd\().b}[11], [\rs], \rt
ld1 {\rd\().b}[12], [\rs], \rt
ld1 {\rd\().b}[13], [\rs], \rt
ld1 {\rd\().b}[14], [\rs], \rt
ld1 {\rd\().b}[15], [\rs], \rt
.endif
.endm
function ff_pred16x16_128_dc_neon, export=1
movi v0.16b, #128
b .L_pred16x16_dc_end
endfunc
function ff_pred16x16_top_dc_neon, export=1
sub x2, x0, x1
ld1 {v0.16b}, [x2]
uaddlv h0, v0.16b
rshrn v0.8b, v0.8h, #4
dup v0.16b, v0.b[0]
b .L_pred16x16_dc_end
endfunc
function ff_pred16x16_left_dc_neon, export=1
sub x2, x0, #1
ldcol.8 v0, x2, x1, 16
uaddlv h0, v0.16b
rshrn v0.8b, v0.8h, #4
dup v0.16b, v0.b[0]
b .L_pred16x16_dc_end
endfunc
function ff_pred16x16_dc_neon, export=1
sub x2, x0, x1
sub x3, x0, #1
ld1 {v0.16b}, [x2]
ldcol.8 v1, x3, x1, 16
uaddlv h0, v0.16b
uaddlv h1, v1.16b
add v0.4h, v0.4h, v1.4h
rshrn v0.8b, v0.8h, #5
dup v0.16b, v0.b[0]
.L_pred16x16_dc_end:
mov w3, #8
6: st1 {v0.16b}, [x0], x1
subs w3, w3, #1
st1 {v0.16b}, [x0], x1
b.ne 6b
ret
endfunc
function ff_pred16x16_hor_neon, export=1
sub x2, x0, #1
mov w3, #16
1: ld1r {v0.16b}, [x2], x1
subs w3, w3, #1
st1 {v0.16b}, [x0], x1
b.ne 1b
ret
endfunc
function ff_pred16x16_vert_neon, export=1
sub x2, x0, x1
add x1, x1, x1
ld1 {v0.16b}, [x2], x1
mov w3, #8
1: subs w3, w3, #1
st1 {v0.16b}, [x0], x1
st1 {v0.16b}, [x2], x1
b.ne 1b
ret
endfunc
function ff_pred16x16_plane_neon, export=1
sub x3, x0, x1
movrel x4, p16weight
add x2, x3, #8
sub x3, x3, #1
ld1 {v0.8b}, [x3]
ld1 {v2.8b}, [x2], x1
ldcol.8 v1, x3, x1
add x3, x3, x1
ldcol.8 v3, x3, x1
rev64 v0.8b, v0.8b
rev64 v1.8b, v1.8b
uaddl v7.8h, v2.8b, v3.8b
usubl v2.8h, v2.8b, v0.8b
usubl v3.8h, v3.8b, v1.8b
ld1 {v0.8h}, [x4]
mul v2.8h, v2.8h, v0.8h
mul v3.8h, v3.8h, v0.8h
addp v2.8h, v2.8h, v3.8h
addp v2.8h, v2.8h, v2.8h
addp v2.4h, v2.4h, v2.4h
sshll v3.4s, v2.4h, #2
saddw v2.4s, v3.4s, v2.4h
rshrn v4.4h, v2.4s, #6
trn2 v5.4h, v4.4h, v4.4h
add v2.4h, v4.4h, v5.4h
shl v3.4h, v2.4h, #3
ext v7.16b, v7.16b, v7.16b, #14
sub v3.4h, v3.4h, v2.4h // 7 * (b + c)
add v7.4h, v7.4h, v0.4h
shl v2.4h, v7.4h, #4
sub v2.4h, v2.4h, v3.4h
shl v3.4h, v4.4h, #4
ext v0.16b, v0.16b, v0.16b, #14
sub v6.4h, v5.4h, v3.4h
mov v0.h[0], wzr
mul v0.8h, v0.8h, v4.h[0]
dup v1.8h, v2.h[0]
dup v2.8h, v4.h[0]
dup v3.8h, v6.h[0]
shl v2.8h, v2.8h, #3
add v1.8h, v1.8h, v0.8h
add v3.8h, v3.8h, v2.8h
mov w3, #16
1:
sqshrun v0.8b, v1.8h, #5
add v1.8h, v1.8h, v2.8h
sqshrun2 v0.16b, v1.8h, #5
add v1.8h, v1.8h, v3.8h
subs w3, w3, #1
st1 {v0.16b}, [x0], x1
b.ne 1b
ret
endfunc
const p16weight, align=4
.short 1,2,3,4,5,6,7,8
endconst
const p8weight, align=4
.short 1,2,3,4,1,2,3,4
endconst
function ff_pred8x8_hor_neon, export=1
sub x2, x0, #1
mov w3, #8
1: ld1r {v0.8b}, [x2], x1
subs w3, w3, #1
st1 {v0.8b}, [x0], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_vert_neon, export=1
sub x2, x0, x1
lsl x1, x1, #1
ld1 {v0.8b}, [x2], x1
mov w3, #4
1: subs w3, w3, #1
st1 {v0.8b}, [x0], x1
st1 {v0.8b}, [x2], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_plane_neon, export=1
sub x3, x0, x1
movrel x4, p8weight
movrel x5, p16weight
add x2, x3, #4
sub x3, x3, #1
ld1 {v0.s}[0], [x3]
ld1 {v2.s}[0], [x2], x1
ldcol.8 v0, x3, x1, 4, hi=1
add x3, x3, x1
ldcol.8 v3, x3, x1, 4
uaddl v7.8h, v2.8b, v3.8b
rev32 v0.8b, v0.8b
trn1 v2.2s, v2.2s, v3.2s
usubl v2.8h, v2.8b, v0.8b
ld1 {v6.8h}, [x4]
mul v2.8h, v2.8h, v6.8h
ld1 {v0.8h}, [x5]
saddlp v2.4s, v2.8h
addp v2.4s, v2.4s, v2.4s
shl v3.4s, v2.4s, #4
add v2.4s, v3.4s, v2.4s
rshrn v5.4h, v2.4s, #5
addp v2.4h, v5.4h, v5.4h
shl v3.4h, v2.4h, #1
add v3.4h, v3.4h, v2.4h
rev64 v7.4h, v7.4h
add v7.4h, v7.4h, v0.4h
shl v2.4h, v7.4h, #4
sub v2.4h, v2.4h, v3.4h
ext v0.16b, v0.16b, v0.16b, #14
mov v0.h[0], wzr
mul v0.8h, v0.8h, v5.h[0]
dup v1.8h, v2.h[0]
dup v2.8h, v5.h[1]
add v1.8h, v1.8h, v0.8h
mov w3, #8
1:
sqshrun v0.8b, v1.8h, #5
subs w3, w3, #1
add v1.8h, v1.8h, v2.8h
st1 {v0.8b}, [x0], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_128_dc_neon, export=1
movi v0.8b, #128
movi v1.8b, #128
b .L_pred8x8_dc_end
endfunc
function ff_pred8x8_top_dc_neon, export=1
sub x2, x0, x1
ld1 {v0.8b}, [x2]
uaddlp v0.4h, v0.8b
addp v0.4h, v0.4h, v0.4h
zip1 v0.8h, v0.8h, v0.8h
rshrn v2.8b, v0.8h, #2
zip1 v0.8b, v2.8b, v2.8b
zip1 v1.8b, v2.8b, v2.8b
b .L_pred8x8_dc_end
endfunc
function ff_pred8x8_left_dc_neon, export=1
sub x2, x0, #1
ldcol.8 v0, x2, x1
uaddlp v0.4h, v0.8b
addp v0.4h, v0.4h, v0.4h
rshrn v2.8b, v0.8h, #2
dup v1.8b, v2.b[1]
dup v0.8b, v2.b[0]
b .L_pred8x8_dc_end
endfunc
function ff_pred8x8_dc_neon, export=1
sub x2, x0, x1
sub x3, x0, #1
ld1 {v0.8b}, [x2]
ldcol.8 v1, x3, x1
uaddlp v0.4h, v0.8b
uaddlp v1.4h, v1.8b
trn1 v2.2s, v0.2s, v1.2s
trn2 v3.2s, v0.2s, v1.2s
addp v4.4h, v2.4h, v3.4h
addp v5.4h, v4.4h, v4.4h
rshrn v6.8b, v5.8h, #3
rshrn v7.8b, v4.8h, #2
dup v0.8b, v6.b[0]
dup v2.8b, v7.b[2]
dup v1.8b, v7.b[3]
dup v3.8b, v6.b[1]
zip1 v0.2s, v0.2s, v2.2s
zip1 v1.2s, v1.2s, v3.2s
.L_pred8x8_dc_end:
mov w3, #4
add x2, x0, x1, lsl #2
6: subs w3, w3, #1
st1 {v0.8b}, [x0], x1
st1 {v1.8b}, [x2], x1
b.ne 6b
ret
endfunc
function ff_pred8x8_l0t_dc_neon, export=1
sub x2, x0, x1
sub x3, x0, #1
ld1 {v0.8b}, [x2]
ldcol.8 v1, x3, x1, 4
zip1 v0.4s, v0.4s, v1.4s
uaddlp v0.8h, v0.16b
addp v0.8h, v0.8h, v0.8h
addp v1.4h, v0.4h, v0.4h
rshrn v2.8b, v0.8h, #2
rshrn v3.8b, v1.8h, #3
dup v4.8b, v3.b[0]
dup v6.8b, v2.b[2]
dup v5.8b, v2.b[0]
zip1 v0.2s, v4.2s, v6.2s
zip1 v1.2s, v5.2s, v6.2s
b .L_pred8x8_dc_end
endfunc
function ff_pred8x8_l00_dc_neon, export=1
sub x2, x0, #1
ldcol.8 v0, x2, x1, 4
uaddlp v0.4h, v0.8b
addp v0.4h, v0.4h, v0.4h
rshrn v0.8b, v0.8h, #2
movi v1.8b, #128
dup v0.8b, v0.b[0]
b .L_pred8x8_dc_end
endfunc
function ff_pred8x8_0lt_dc_neon, export=1
add x3, x0, x1, lsl #2
sub x2, x0, x1
sub x3, x3, #1
ld1 {v0.8b}, [x2]
ldcol.8 v1, x3, x1, 4, hi=1
zip1 v0.4s, v0.4s, v1.4s
uaddlp v0.8h, v0.16b
addp v0.8h, v0.8h, v0.8h
addp v1.4h, v0.4h, v0.4h
rshrn v2.8b, v0.8h, #2
rshrn v3.8b, v1.8h, #3
dup v4.8b, v2.b[0]
dup v5.8b, v2.b[3]
dup v6.8b, v2.b[2]
dup v7.8b, v3.b[1]
zip1 v0.2s, v4.2s, v6.2s
zip1 v1.2s, v5.2s, v7.2s
b .L_pred8x8_dc_end
endfunc
function ff_pred8x8_0l0_dc_neon, export=1
add x2, x0, x1, lsl #2
sub x2, x2, #1
ldcol.8 v1, x2, x1, 4
uaddlp v2.4h, v1.8b
addp v2.4h, v2.4h, v2.4h
rshrn v1.8b, v2.8h, #2
movi v0.8b, #128
dup v1.8b, v1.b[0]
b .L_pred8x8_dc_end
endfunc
.macro ldcol.16 rd, rs, rt, n=4, hi=0
.if \n >= 4 && \hi == 0
ld1 {\rd\().h}[0], [\rs], \rt
ld1 {\rd\().h}[1], [\rs], \rt
ld1 {\rd\().h}[2], [\rs], \rt
ld1 {\rd\().h}[3], [\rs], \rt
.endif
.if \n == 8 || \hi == 1
ld1 {\rd\().h}[4], [\rs], \rt
ld1 {\rd\().h}[5], [\rs], \rt
ld1 {\rd\().h}[6], [\rs], \rt
ld1 {\rd\().h}[7], [\rs], \rt
.endif
.endm
// slower than C
/*
function ff_pred16x16_128_dc_neon_10, export=1
movi v0.8h, #2, lsl #8 // 512, 1 << (bit_depth - 1)
b .L_pred16x16_dc_10_end
endfunc
*/
function ff_pred16x16_top_dc_neon_10, export=1
sub x2, x0, x1
ld1 {v0.8h, v1.8h}, [x2]
add v0.8h, v0.8h, v1.8h
addv h0, v0.8h
urshr v0.4h, v0.4h, #4
dup v0.8h, v0.h[0]
b .L_pred16x16_dc_10_end
endfunc
// slower than C
/*
function ff_pred16x16_left_dc_neon_10, export=1
sub x2, x0, #2 // access to the "left" column
ldcol.16 v0, x2, x1, 8
ldcol.16 v1, x2, x1, 8 // load "left" column
add v0.8h, v0.8h, v1.8h
addv h0, v0.8h
urshr v0.4h, v0.4h, #4
dup v0.8h, v0.h[0]
b .L_pred16x16_dc_10_end
endfunc
*/
function ff_pred16x16_dc_neon_10, export=1
sub x2, x0, x1 // access to the "top" row
sub x3, x0, #2 // access to the "left" column
ld1 {v0.8h, v1.8h}, [x2]
ldcol.16 v2, x3, x1, 8
ldcol.16 v3, x3, x1, 8 // load pixels in "top" row and "left" col
add v0.8h, v0.8h, v1.8h
add v2.8h, v2.8h, v3.8h
add v0.8h, v0.8h, v2.8h
addv h0, v0.8h
urshr v0.4h, v0.4h, #5
dup v0.8h, v0.h[0]
.L_pred16x16_dc_10_end:
mov v1.16b, v0.16b
mov w3, #8
6: st1 {v0.8h, v1.8h}, [x0], x1
subs w3, w3, #1
st1 {v0.8h, v1.8h}, [x0], x1
b.ne 6b
ret
endfunc
function ff_pred16x16_hor_neon_10, export=1
sub x2, x0, #2
add x3, x0, #16
mov w4, #16
1: ld1r {v0.8h}, [x2], x1
subs w4, w4, #1
st1 {v0.8h}, [x0], x1
st1 {v0.8h}, [x3], x1
b.ne 1b
ret
endfunc
function ff_pred16x16_vert_neon_10, export=1
sub x2, x0, x1
add x1, x1, x1
ld1 {v0.8h, v1.8h}, [x2], x1
mov w3, #8
1: subs w3, w3, #1
st1 {v0.8h, v1.8h}, [x0], x1
st1 {v0.8h, v1.8h}, [x2], x1
b.ne 1b
ret
endfunc
function ff_pred16x16_plane_neon_10, export=1
sub x3, x0, x1
movrel x4, p16weight
add x2, x3, #16
sub x3, x3, #2
ld1 {v0.8h}, [x3]
ld1 {v2.8h}, [x2], x1
ldcol.16 v1, x3, x1, 8
add x3, x3, x1
ldcol.16 v3, x3, x1, 8
rev64 v16.8h, v0.8h
rev64 v17.8h, v1.8h
ext v0.16b, v16.16b, v16.16b, #8
ext v1.16b, v17.16b, v17.16b, #8
add v7.8h, v2.8h, v3.8h
sub v2.8h, v2.8h, v0.8h
sub v3.8h, v3.8h, v1.8h
ld1 {v0.8h}, [x4]
mul v2.8h, v2.8h, v0.8h
mul v3.8h, v3.8h, v0.8h
addp v2.8h, v2.8h, v3.8h
saddlp v2.4s, v2.8h
addp v2.4s, v2.4s, v2.4s
shl v3.4s, v2.4s, #2
add v2.4s, v3.4s, v2.4s
rshrn v4.4h, v2.4s, #6
trn2 v5.4h, v4.4h, v4.4h
add v2.4h, v4.4h, v5.4h
shl v3.4h, v2.4h, #3
ext v7.16b, v7.16b, v7.16b, #14
sub v3.4h, v3.4h, v2.4h // 7 * (b + c)
add v7.4h, v7.4h, v0.4h
shl v2.4h, v7.4h, #4
ssubl v2.4s, v2.4h, v3.4h
ext v0.16b, v0.16b, v0.16b, #14
sxtl v6.4s, v5.4h // c
mov v0.h[0], wzr
dup v16.4s, v2.s[0]
dup v17.4s, v2.s[0]
dup v2.8h, v4.h[0] // b
dup v3.4s, v6.s[0] // c
sshll v2.4s, v2.4h, #3 // b * 8
smlal v16.4s, v0.4h, v4.h[0]
smlal2 v17.4s, v0.8h, v4.h[0]
sub v3.4s, v3.4s, v2.4s
mov w3, #16
mvni v4.8h, #0xFC, lsl #8 // 1023 for clipping
1:
sqshrun v0.4h, v16.4s, #5
sqshrun2 v0.8h, v17.4s, #5
add v16.4s, v16.4s, v2.4s
add v17.4s, v17.4s, v2.4s
sqshrun v1.4h, v16.4s, #5
sqshrun2 v1.8h, v17.4s, #5
add v16.4s, v16.4s, v3.4s
add v17.4s, v17.4s, v3.4s
subs w3, w3, #1
smin v0.8h, v0.8h, v4.8h
smin v1.8h, v1.8h, v4.8h
st1 {v0.8h, v1.8h}, [x0], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_hor_neon_10, export=1
sub x2, x0, #2
mov w3, #8
1: ld1r {v0.8h}, [x2], x1
subs w3, w3, #1
st1 {v0.8h}, [x0], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_vert_neon_10, export=1
sub x2, x0, x1
lsl x1, x1, #1
ld1 {v0.8h}, [x2], x1
mov w3, #4
1: subs w3, w3, #1
st1 {v0.8h}, [x0], x1
st1 {v0.8h}, [x2], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_plane_neon_10, export=1
sub x3, x0, x1
movrel x4, p8weight
movrel x5, p16weight
add x2, x3, #8
sub x3, x3, #2
ld1 {v0.d}[0], [x3]
ld1 {v2.d}[0], [x2], x1
ldcol.16 v0, x3, x1, hi=1
add x3, x3, x1
ldcol.16 v3, x3, x1, 4
add v7.8h, v2.8h, v3.8h
rev64 v0.8h, v0.8h
trn1 v2.2d, v2.2d, v3.2d
sub v2.8h, v2.8h, v0.8h
ld1 {v6.8h}, [x4]
mul v2.8h, v2.8h, v6.8h
ld1 {v0.8h}, [x5]
saddlp v2.4s, v2.8h
addp v2.4s, v2.4s, v2.4s
shl v3.4s, v2.4s, #4
add v2.4s, v3.4s, v2.4s
rshrn v5.4h, v2.4s, #5
addp v2.4h, v5.4h, v5.4h
shl v3.4h, v2.4h, #1
add v3.4h, v3.4h, v2.4h
rev64 v7.4h, v7.4h
add v7.4h, v7.4h, v0.4h
shl v2.4h, v7.4h, #4
ssubl v2.4s, v2.4h, v3.4h
ext v0.16b, v0.16b, v0.16b, #14
mov v0.h[0], wzr
dup v1.4s, v2.s[0]
dup v2.4s, v2.s[0]
dup v3.8h, v5.h[1]
smlal v1.4s, v0.4h, v5.h[0]
smlal2 v2.4s, v0.8h, v5.h[0]
mov w3, #8
mvni v4.8h, #0xFC, lsl #8 // 1023 for clipping
1:
sqshrun v0.4h, v1.4s, #5
sqshrun2 v0.8h, v2.4s, #5
saddw v1.4s, v1.4s, v3.4h
saddw v2.4s, v2.4s, v3.4h
subs w3, w3, #1
smin v0.8h, v0.8h, v4.8h
st1 {v0.8h}, [x0], x1
b.ne 1b
ret
endfunc
function ff_pred8x8_128_dc_neon_10, export=1
movi v0.8h, #2, lsl #8 // 512, 1 << (bit_depth - 1)
movi v1.8h, #2, lsl #8
b .L_pred8x8_dc_10_end
endfunc
function ff_pred8x8_top_dc_neon_10, export=1
sub x2, x0, x1
ld1 {v0.8h}, [x2]
addp v0.8h, v0.8h, v0.8h
addp v0.4h, v0.4h, v0.4h
zip1 v0.4h, v0.4h, v0.4h
urshr v2.4h, v0.4h, #2
zip1 v0.8h, v2.8h, v2.8h
zip1 v1.8h, v2.8h, v2.8h
b .L_pred8x8_dc_10_end
endfunc
function ff_pred8x8_left_dc_neon_10, export=1
sub x2, x0, #2
ldcol.16 v0, x2, x1, 8
addp v0.8h, v0.8h, v0.8h
addp v0.4h, v0.4h, v0.4h
urshr v2.4h, v0.4h, #2
dup v1.8h, v2.h[1]
dup v0.8h, v2.h[0]
b .L_pred8x8_dc_10_end
endfunc
function ff_pred8x8_dc_neon_10, export=1
sub x2, x0, x1
sub x3, x0, #2
ld1 {v0.8h}, [x2]
ldcol.16 v1, x3, x1, 8
addp v0.8h, v0.8h, v0.8h
addp v1.8h, v1.8h, v1.8h
trn1 v2.2s, v0.2s, v1.2s
trn2 v3.2s, v0.2s, v1.2s
addp v4.4h, v2.4h, v3.4h
addp v5.4h, v4.4h, v4.4h
urshr v6.4h, v5.4h, #3
urshr v7.4h, v4.4h, #2
dup v0.8h, v6.h[0]
dup v2.8h, v7.h[2]
dup v1.8h, v7.h[3]
dup v3.8h, v6.h[1]
zip1 v0.2d, v0.2d, v2.2d
zip1 v1.2d, v1.2d, v3.2d
.L_pred8x8_dc_10_end:
mov w3, #4
add x2, x0, x1, lsl #2
6: st1 {v0.8h}, [x0], x1
subs w3, w3, #1
st1 {v1.8h}, [x2], x1
b.ne 6b
ret
endfunc
function ff_pred8x8_l0t_dc_neon_10, export=1
sub x2, x0, x1
sub x3, x0, #2
ld1 {v0.8h}, [x2]
ldcol.16 v1, x3, x1, 4
addp v0.8h, v0.8h, v0.8h
addp v1.4h, v1.4h, v1.4h
addp v0.4h, v0.4h, v0.4h
addp v1.4h, v1.4h, v1.4h
add v1.4h, v1.4h, v0.4h
urshr v2.4h, v0.4h, #2
urshr v3.4h, v1.4h, #3 // the pred4x4 part
dup v4.4h, v3.h[0]
dup v5.4h, v2.h[0]
dup v6.4h, v2.h[1]
zip1 v0.2d, v4.2d, v6.2d
zip1 v1.2d, v5.2d, v6.2d
b .L_pred8x8_dc_10_end
endfunc
function ff_pred8x8_l00_dc_neon_10, export=1
sub x2, x0, #2
ldcol.16 v0, x2, x1, 4
addp v0.4h, v0.4h, v0.4h
addp v0.4h, v0.4h, v0.4h
urshr v0.4h, v0.4h, #2
movi v1.8h, #2, lsl #8 // 512
dup v0.8h, v0.h[0]
b .L_pred8x8_dc_10_end
endfunc
function ff_pred8x8_0lt_dc_neon_10, export=1
add x3, x0, x1, lsl #2
sub x2, x0, x1
sub x3, x3, #2
ld1 {v0.8h}, [x2]
ldcol.16 v1, x3, x1, hi=1
addp v0.8h, v0.8h, v0.8h
addp v1.8h, v1.8h, v1.8h
addp v0.4h, v0.4h, v0.4h
addp v1.4h, v1.4h, v1.4h
zip1 v0.2s, v0.2s, v1.2s
add v1.4h, v0.4h, v1.4h
urshr v2.4h, v0.4h, #2
urshr v3.4h, v1.4h, #3
dup v4.4h, v2.h[0]
dup v5.4h, v2.h[3]
dup v6.4h, v2.h[1]
dup v7.4h, v3.h[1]
zip1 v0.2d, v4.2d, v6.2d
zip1 v1.2d, v5.2d, v7.2d
b .L_pred8x8_dc_10_end
endfunc
function ff_pred8x8_0l0_dc_neon_10, export=1
add x2, x0, x1, lsl #2
sub x2, x2, #2
ldcol.16 v1, x2, x1, 4
addp v2.8h, v1.8h, v1.8h
addp v2.4h, v2.4h, v2.4h
urshr v1.4h, v2.4h, #2
movi v0.8h, #2, lsl #8 // 512
dup v1.8h, v1.h[0]
b .L_pred8x8_dc_10_end
endfunc
@@ -0,0 +1,259 @@
/*
* ARM NEON optimised DSP functions
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/h264qpel.h"
void ff_put_h264_qpel16_mc00_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc10_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc20_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc30_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc01_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc11_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc21_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc31_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc02_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc12_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc22_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc32_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc03_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc13_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc23_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc33_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc00_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc10_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc20_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc30_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc01_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc11_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc21_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc31_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc02_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc12_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc22_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc32_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc03_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc13_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc23_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc33_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc00_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc10_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc20_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc30_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc01_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc11_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc21_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc31_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc02_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc12_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc22_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc32_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc03_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc13_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc23_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc33_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc00_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc10_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc20_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc30_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc01_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc11_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc21_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc31_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc02_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc12_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc22_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc32_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc03_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc13_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc23_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc33_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc10_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc20_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc30_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc01_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc11_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc31_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc02_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc03_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc13_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel16_mc33_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc10_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc20_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc30_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc01_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc11_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc31_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc02_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc03_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc13_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_put_h264_qpel8_mc33_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc10_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc20_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc30_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc01_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc11_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc31_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc02_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc03_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc13_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel16_mc33_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc10_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc20_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc30_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc01_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc11_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc31_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc02_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc03_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc13_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
void ff_avg_h264_qpel8_mc33_neon_10(uint8_t *dst, const uint8_t *src, ptrdiff_t stride);
av_cold void ff_h264qpel_init_aarch64(H264QpelContext *c, int bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags) && bit_depth <= 8) {
c->put_h264_qpel_pixels_tab[0][ 0] = ff_put_h264_qpel16_mc00_neon;
c->put_h264_qpel_pixels_tab[0][ 1] = ff_put_h264_qpel16_mc10_neon;
c->put_h264_qpel_pixels_tab[0][ 2] = ff_put_h264_qpel16_mc20_neon;
c->put_h264_qpel_pixels_tab[0][ 3] = ff_put_h264_qpel16_mc30_neon;
c->put_h264_qpel_pixels_tab[0][ 4] = ff_put_h264_qpel16_mc01_neon;
c->put_h264_qpel_pixels_tab[0][ 5] = ff_put_h264_qpel16_mc11_neon;
c->put_h264_qpel_pixels_tab[0][ 6] = ff_put_h264_qpel16_mc21_neon;
c->put_h264_qpel_pixels_tab[0][ 7] = ff_put_h264_qpel16_mc31_neon;
c->put_h264_qpel_pixels_tab[0][ 8] = ff_put_h264_qpel16_mc02_neon;
c->put_h264_qpel_pixels_tab[0][ 9] = ff_put_h264_qpel16_mc12_neon;
c->put_h264_qpel_pixels_tab[0][10] = ff_put_h264_qpel16_mc22_neon;
c->put_h264_qpel_pixels_tab[0][11] = ff_put_h264_qpel16_mc32_neon;
c->put_h264_qpel_pixels_tab[0][12] = ff_put_h264_qpel16_mc03_neon;
c->put_h264_qpel_pixels_tab[0][13] = ff_put_h264_qpel16_mc13_neon;
c->put_h264_qpel_pixels_tab[0][14] = ff_put_h264_qpel16_mc23_neon;
c->put_h264_qpel_pixels_tab[0][15] = ff_put_h264_qpel16_mc33_neon;
c->put_h264_qpel_pixels_tab[1][ 0] = ff_put_h264_qpel8_mc00_neon;
c->put_h264_qpel_pixels_tab[1][ 1] = ff_put_h264_qpel8_mc10_neon;
c->put_h264_qpel_pixels_tab[1][ 2] = ff_put_h264_qpel8_mc20_neon;
c->put_h264_qpel_pixels_tab[1][ 3] = ff_put_h264_qpel8_mc30_neon;
c->put_h264_qpel_pixels_tab[1][ 4] = ff_put_h264_qpel8_mc01_neon;
c->put_h264_qpel_pixels_tab[1][ 5] = ff_put_h264_qpel8_mc11_neon;
c->put_h264_qpel_pixels_tab[1][ 6] = ff_put_h264_qpel8_mc21_neon;
c->put_h264_qpel_pixels_tab[1][ 7] = ff_put_h264_qpel8_mc31_neon;
c->put_h264_qpel_pixels_tab[1][ 8] = ff_put_h264_qpel8_mc02_neon;
c->put_h264_qpel_pixels_tab[1][ 9] = ff_put_h264_qpel8_mc12_neon;
c->put_h264_qpel_pixels_tab[1][10] = ff_put_h264_qpel8_mc22_neon;
c->put_h264_qpel_pixels_tab[1][11] = ff_put_h264_qpel8_mc32_neon;
c->put_h264_qpel_pixels_tab[1][12] = ff_put_h264_qpel8_mc03_neon;
c->put_h264_qpel_pixels_tab[1][13] = ff_put_h264_qpel8_mc13_neon;
c->put_h264_qpel_pixels_tab[1][14] = ff_put_h264_qpel8_mc23_neon;
c->put_h264_qpel_pixels_tab[1][15] = ff_put_h264_qpel8_mc33_neon;
c->avg_h264_qpel_pixels_tab[0][ 0] = ff_avg_h264_qpel16_mc00_neon;
c->avg_h264_qpel_pixels_tab[0][ 1] = ff_avg_h264_qpel16_mc10_neon;
c->avg_h264_qpel_pixels_tab[0][ 2] = ff_avg_h264_qpel16_mc20_neon;
c->avg_h264_qpel_pixels_tab[0][ 3] = ff_avg_h264_qpel16_mc30_neon;
c->avg_h264_qpel_pixels_tab[0][ 4] = ff_avg_h264_qpel16_mc01_neon;
c->avg_h264_qpel_pixels_tab[0][ 5] = ff_avg_h264_qpel16_mc11_neon;
c->avg_h264_qpel_pixels_tab[0][ 6] = ff_avg_h264_qpel16_mc21_neon;
c->avg_h264_qpel_pixels_tab[0][ 7] = ff_avg_h264_qpel16_mc31_neon;
c->avg_h264_qpel_pixels_tab[0][ 8] = ff_avg_h264_qpel16_mc02_neon;
c->avg_h264_qpel_pixels_tab[0][ 9] = ff_avg_h264_qpel16_mc12_neon;
c->avg_h264_qpel_pixels_tab[0][10] = ff_avg_h264_qpel16_mc22_neon;
c->avg_h264_qpel_pixels_tab[0][11] = ff_avg_h264_qpel16_mc32_neon;
c->avg_h264_qpel_pixels_tab[0][12] = ff_avg_h264_qpel16_mc03_neon;
c->avg_h264_qpel_pixels_tab[0][13] = ff_avg_h264_qpel16_mc13_neon;
c->avg_h264_qpel_pixels_tab[0][14] = ff_avg_h264_qpel16_mc23_neon;
c->avg_h264_qpel_pixels_tab[0][15] = ff_avg_h264_qpel16_mc33_neon;
c->avg_h264_qpel_pixels_tab[1][ 0] = ff_avg_h264_qpel8_mc00_neon;
c->avg_h264_qpel_pixels_tab[1][ 1] = ff_avg_h264_qpel8_mc10_neon;
c->avg_h264_qpel_pixels_tab[1][ 2] = ff_avg_h264_qpel8_mc20_neon;
c->avg_h264_qpel_pixels_tab[1][ 3] = ff_avg_h264_qpel8_mc30_neon;
c->avg_h264_qpel_pixels_tab[1][ 4] = ff_avg_h264_qpel8_mc01_neon;
c->avg_h264_qpel_pixels_tab[1][ 5] = ff_avg_h264_qpel8_mc11_neon;
c->avg_h264_qpel_pixels_tab[1][ 6] = ff_avg_h264_qpel8_mc21_neon;
c->avg_h264_qpel_pixels_tab[1][ 7] = ff_avg_h264_qpel8_mc31_neon;
c->avg_h264_qpel_pixels_tab[1][ 8] = ff_avg_h264_qpel8_mc02_neon;
c->avg_h264_qpel_pixels_tab[1][ 9] = ff_avg_h264_qpel8_mc12_neon;
c->avg_h264_qpel_pixels_tab[1][10] = ff_avg_h264_qpel8_mc22_neon;
c->avg_h264_qpel_pixels_tab[1][11] = ff_avg_h264_qpel8_mc32_neon;
c->avg_h264_qpel_pixels_tab[1][12] = ff_avg_h264_qpel8_mc03_neon;
c->avg_h264_qpel_pixels_tab[1][13] = ff_avg_h264_qpel8_mc13_neon;
c->avg_h264_qpel_pixels_tab[1][14] = ff_avg_h264_qpel8_mc23_neon;
c->avg_h264_qpel_pixels_tab[1][15] = ff_avg_h264_qpel8_mc33_neon;
} else if (have_neon(cpu_flags) && bit_depth == 10) {
c->put_h264_qpel_pixels_tab[0][ 1] = ff_put_h264_qpel16_mc10_neon_10;
c->put_h264_qpel_pixels_tab[0][ 2] = ff_put_h264_qpel16_mc20_neon_10;
c->put_h264_qpel_pixels_tab[0][ 3] = ff_put_h264_qpel16_mc30_neon_10;
c->put_h264_qpel_pixels_tab[0][ 4] = ff_put_h264_qpel16_mc01_neon_10;
c->put_h264_qpel_pixels_tab[0][ 5] = ff_put_h264_qpel16_mc11_neon_10;
c->put_h264_qpel_pixels_tab[0][ 7] = ff_put_h264_qpel16_mc31_neon_10;
c->put_h264_qpel_pixels_tab[0][ 8] = ff_put_h264_qpel16_mc02_neon_10;
c->put_h264_qpel_pixels_tab[0][12] = ff_put_h264_qpel16_mc03_neon_10;
c->put_h264_qpel_pixels_tab[0][13] = ff_put_h264_qpel16_mc13_neon_10;
c->put_h264_qpel_pixels_tab[0][15] = ff_put_h264_qpel16_mc33_neon_10;
c->put_h264_qpel_pixels_tab[1][ 1] = ff_put_h264_qpel8_mc10_neon_10;
c->put_h264_qpel_pixels_tab[1][ 2] = ff_put_h264_qpel8_mc20_neon_10;
c->put_h264_qpel_pixels_tab[1][ 3] = ff_put_h264_qpel8_mc30_neon_10;
c->put_h264_qpel_pixels_tab[1][ 4] = ff_put_h264_qpel8_mc01_neon_10;
c->put_h264_qpel_pixels_tab[1][ 5] = ff_put_h264_qpel8_mc11_neon_10;
c->put_h264_qpel_pixels_tab[1][ 7] = ff_put_h264_qpel8_mc31_neon_10;
c->put_h264_qpel_pixels_tab[1][ 8] = ff_put_h264_qpel8_mc02_neon_10;
c->put_h264_qpel_pixels_tab[1][12] = ff_put_h264_qpel8_mc03_neon_10;
c->put_h264_qpel_pixels_tab[1][13] = ff_put_h264_qpel8_mc13_neon_10;
c->put_h264_qpel_pixels_tab[1][15] = ff_put_h264_qpel8_mc33_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 1] = ff_avg_h264_qpel16_mc10_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 2] = ff_avg_h264_qpel16_mc20_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 3] = ff_avg_h264_qpel16_mc30_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 4] = ff_avg_h264_qpel16_mc01_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 5] = ff_avg_h264_qpel16_mc11_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 7] = ff_avg_h264_qpel16_mc31_neon_10;
c->avg_h264_qpel_pixels_tab[0][ 8] = ff_avg_h264_qpel16_mc02_neon_10;
c->avg_h264_qpel_pixels_tab[0][12] = ff_avg_h264_qpel16_mc03_neon_10;
c->avg_h264_qpel_pixels_tab[0][13] = ff_avg_h264_qpel16_mc13_neon_10;
c->avg_h264_qpel_pixels_tab[0][15] = ff_avg_h264_qpel16_mc33_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 1] = ff_avg_h264_qpel8_mc10_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 2] = ff_avg_h264_qpel8_mc20_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 3] = ff_avg_h264_qpel8_mc30_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 4] = ff_avg_h264_qpel8_mc01_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 5] = ff_avg_h264_qpel8_mc11_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 7] = ff_avg_h264_qpel8_mc31_neon_10;
c->avg_h264_qpel_pixels_tab[1][ 8] = ff_avg_h264_qpel8_mc02_neon_10;
c->avg_h264_qpel_pixels_tab[1][12] = ff_avg_h264_qpel8_mc03_neon_10;
c->avg_h264_qpel_pixels_tab[1][13] = ff_avg_h264_qpel8_mc13_neon_10;
c->avg_h264_qpel_pixels_tab[1][15] = ff_avg_h264_qpel8_mc33_neon_10;
}
}
File diff suppressed because it is too large. Load diff
+308
View File
@@ -0,0 +1,308 @@
/*
* Copyright (C) 2024 Zhao Zhili
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_AARCH64_H26X_DSP_H
#define AVCODEC_AARCH64_H26X_DSP_H
#include <stddef.h>
#include <stdint.h>
void ff_h26x_sao_band_filter_8x8_8_neon(uint8_t *_dst, const uint8_t *_src,
ptrdiff_t stride_dst, ptrdiff_t stride_src,
const int16_t *sao_offset_val, int sao_left_class,
int width, int height);
void ff_hevc_sao_edge_filter_16x16_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst,
const int16_t *sao_offset_val, int eo, int width, int height);
void ff_hevc_sao_edge_filter_8x8_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst,
const int16_t *sao_offset_val, int eo, int width, int height);
void ff_vvc_sao_edge_filter_16x16_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst,
const int16_t *sao_offset_val, int eo, int width, int height);
void ff_vvc_sao_edge_filter_8x8_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst,
const int16_t *sao_offset_val, int eo, int width, int height);
#define NEON8_FNPROTO_PARTIAL_6(fn, args, ext) \
void ff_hevc_put_hevc_##fn##_h4_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##_h6_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##_h8_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##_h12_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##_h16_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##_h32_8_neon##ext args;
NEON8_FNPROTO_PARTIAL_6(qpel, (int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
intptr_t mx, intptr_t my, int width),)
NEON8_FNPROTO_PARTIAL_6(qpel_uni, (uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width),)
NEON8_FNPROTO_PARTIAL_6(qpel_bi, (uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
ptrdiff_t _srcstride, const int16_t *src2, int height, intptr_t
mx, intptr_t my, int width),)
#define NEON8_FNPROTO(fn, args, ext) \
void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##6_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##12_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##24_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##32_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##48_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##64_8_neon##ext args
#define NEON8_FNPROTO_PARTIAL_4(fn, args, ext) \
void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##64_8_neon##ext args
#define NEON8_FNPROTO_PARTIAL_5(fn, args, ext) \
void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##32_8_neon##ext args; \
void ff_hevc_put_hevc_##fn##64_8_neon##ext args
NEON8_FNPROTO(pel_pixels, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(pel_bi_pixels, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *_src, ptrdiff_t _srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_bi_h, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_bi_v, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(epel_v, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(pel_uni_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(pel_uni_w_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_uni_v, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_uni_hv, (uint8_t *dst, ptrdiff_t _dststride,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_uni_hv, (uint8_t *dst, ptrdiff_t _dststride,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(epel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO_PARTIAL_4(qpel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_h, (int16_t *dst,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width), );
NEON8_FNPROTO(epel_h, (int16_t *dst,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(epel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(qpel_h, (int16_t *dst,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(qpel_v, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(qpel_uni_v, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_uni_hv, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_uni_hv, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(qpel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(epel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(epel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
intptr_t mx, intptr_t my, int width), _i8mm);
NEON8_FNPROTO(qpel_bi_v, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width),);
NEON8_FNPROTO(qpel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
const uint8_t *src, ptrdiff_t srcstride, const int16_t *src2,
int height, intptr_t mx, intptr_t my, int width), _i8mm);
#undef NEON8_FNPROTO_PARTIAL_4
#define NEON8_FNPROTO_PARTIAL_4(fn, args, ext) \
void ff_vvc_put_##fn##_h4_8_neon##ext args; \
void ff_vvc_put_##fn##_h8_8_neon##ext args; \
void ff_vvc_put_##fn##_h16_8_neon##ext args; \
void ff_vvc_put_##fn##_h32_8_neon##ext args;
NEON8_FNPROTO_PARTIAL_4(qpel, (int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width),)
NEON8_FNPROTO_PARTIAL_4(qpel_uni, (uint8_t *_dst, ptrdiff_t _dststride, const uint8_t *_src,
ptrdiff_t _srcstride, int height, const int8_t *hf, const int8_t *vf, int width),)
NEON8_FNPROTO_PARTIAL_4(epel, (int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width),)
#undef NEON8_FNPROTO_PARTIAL_6
#define NEON8_FNPROTO_PARTIAL_6(fn, args, ext) \
void ff_vvc_put_##fn##4_8_neon##ext args; \
void ff_vvc_put_##fn##8_8_neon##ext args; \
void ff_vvc_put_##fn##16_8_neon##ext args; \
void ff_vvc_put_##fn##32_8_neon##ext args; \
void ff_vvc_put_##fn##64_8_neon##ext args; \
void ff_vvc_put_##fn##128_8_neon##ext args
NEON8_FNPROTO_PARTIAL_6(pel_pixels, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride, int height,
const int8_t *hf, const int8_t *vf, int width),);
NEON8_FNPROTO_PARTIAL_6(pel_uni_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width),);
NEON8_FNPROTO_PARTIAL_6(pel_uni_w_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
const uint8_t *_src, ptrdiff_t _srcstride,
int height, int denom, int wx, int ox,
const int8_t *hf, const int8_t *vf, int width),);
NEON8_FNPROTO_PARTIAL_6(qpel_h, (int16_t *dst,
const uint8_t *_src, ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width), _i8mm);
NEON8_FNPROTO_PARTIAL_6(epel_h, (int16_t *dst,
const uint8_t *_src, ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width), _i8mm);
void ff_vvc_put_qpel_v4_8_neon(int16_t *dst, const uint8_t *_src,
ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width);
void ff_vvc_put_qpel_v8_8_neon(int16_t *dst, const uint8_t *_src,
ptrdiff_t _srcstride, int height,
const int8_t *hf, const int8_t *vf, int width);
NEON8_FNPROTO_PARTIAL_6(qpel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride, int height,
const int8_t *hf, const int8_t *vf, int width),);
NEON8_FNPROTO_PARTIAL_6(qpel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride, int height,
const int8_t *hf, const int8_t *vf, int width), _i8mm);
NEON8_FNPROTO_PARTIAL_6(epel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride, int height,
const int8_t *hf, const int8_t *vf, int width),);
NEON8_FNPROTO_PARTIAL_6(epel_hv, (int16_t *dst,
const uint8_t *src, ptrdiff_t srcstride, int height,
const int8_t *hf, const int8_t *vf, int width), _i8mm);
#endif
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+218
View File
@@ -0,0 +1,218 @@
/* -*-arm64-*-
* vim: syntax=arm64asm
*
* AArch64 NEON optimised SAO functions for h26x decoding
*
* Copyright (c) 2022 J. Dekker <jdek@itanimul.li>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#define HEVC_MAX_PB_SIZE 64
#define VVC_MAX_PB_SIZE 128
#define AV_INPUT_BUFFER_PADDING_SIZE 64
#define HEVC_SAO_STRIDE (2 * HEVC_MAX_PB_SIZE + AV_INPUT_BUFFER_PADDING_SIZE)
#define VVC_SAO_STRIDE (2 * VVC_MAX_PB_SIZE + AV_INPUT_BUFFER_PADDING_SIZE)
// void sao_band_filter(uint8_t *_dst, uint8_t *_src,
// ptrdiff_t stride_dst, ptrdiff_t stride_src,
// int16_t *sao_offset_val, int sao_left_class,
// int width, int height)
function ff_h26x_sao_band_filter_8x8_8_neon, export=1
stp xzr, xzr, [sp, #-64]!
stp xzr, xzr, [sp, #16]
stp xzr, xzr, [sp, #32]
stp xzr, xzr, [sp, #48]
mov w8, #4
0: ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1]
subs w8, w8, #1
add w10, w8, w5 // k + sao_left_class
and w10, w10, #0x1F
strh w9, [sp, x10, lsl #1]
bne 0b
add w6, w6, #7
bic w6, w6, #7
ld1 {v16.16b-v19.16b}, [sp], #64
sub x2, x2, x6
sub x3, x3, x6
movi v20.8h, #1
1: mov w8, w6 // beginning of line
2: // Simple layout for accessing 16bit values
// with 8bit LUT.
//
// 00 01 02 03 04 05 06 07
// +----------------------------------->
// |xDE#xAD|xCA#xFE|xBE#xEF|xFE#xED|....
// +----------------------------------->
// i-0 i-1 i-2 i-3
ld1 {v2.8b}, [x1], #8 // dst[x] = av_clip_pixel(src[x] + offset_table[src[x] >> shift]);
subs w8, w8, #8
uxtl v0.8h, v2.8b // load src[x]
ushr v2.8h, v0.8h, #3 // >> BIT_DEPTH - 3
shl v1.8h, v2.8h, #1 // low (x2, accessing short)
add v3.8h, v1.8h, v20.8h // +1 access upper short
sli v1.8h, v3.8h, #8 // shift insert index to upper byte
tbx v2.16b, {v16.16b-v19.16b}, v1.16b // table
add v1.8h, v0.8h, v2.8h // src[x] + table
sqxtun v4.8b, v1.8h // clip + narrow
st1 {v4.8b}, [x0], #8 // store
// done 8 pixels
bne 2b
subs w7, w7, #1 // finished line, prep. new
add x0, x0, x2 // dst += stride_dst
add x1, x1, x3 // src += stride_src
bne 1b
ret
endfunc
.Lhevc_sao_edge_pos:
.word 1 // horizontal
.word HEVC_SAO_STRIDE // vertical
.word HEVC_SAO_STRIDE + 1 // 45 degree
.word HEVC_SAO_STRIDE - 1 // 135 degree
.Lvvc_sao_edge_pos:
.word 1 // horizontal
.word VVC_SAO_STRIDE // vertical
.word VVC_SAO_STRIDE + 1 // 45 degree
.word VVC_SAO_STRIDE - 1 // 135 degree
function ff_vvc_sao_edge_filter_16x16_8_neon, export=1
adr x7, .Lvvc_sao_edge_pos
mov x15, #VVC_SAO_STRIDE
b 1f
endfunc
// ff_hevc_sao_edge_filter_16x16_8_neon(char *dst, char *src, ptrdiff stride_dst,
// int16 *sao_offset_val, int eo, int width, int height)
function ff_hevc_sao_edge_filter_16x16_8_neon, export=1
adr x7, .Lhevc_sao_edge_pos
mov x15, #HEVC_SAO_STRIDE
1:
ld1 {v3.8h}, [x3] // load sao_offset_val
add w5, w5, #0xF
bic w5, w5, #0xF
ldr w4, [x7, w4, uxtw #2] // stride_src
mov v3.h[7], v3.h[0] // reorder to [1,2,0,3,4]
mov v3.h[0], v3.h[1]
mov v3.h[1], v3.h[2]
mov v3.h[2], v3.h[7]
// split 16bit values into two tables
uzp2 v1.16b, v3.16b, v3.16b // sao_offset_val -> upper
uzp1 v0.16b, v3.16b, v3.16b // sao_offset_val -> lower
movi v2.16b, #2
// strides between end of line and next src/dst
sub x15, x15, x5 // stride_src - width
sub x16, x2, x5 // stride_dst - width
mov x11, x1 // copy base src
1: // new line
mov x14, x5 // copy width
sub x12, x11, x4 // src_a (prev) = src - sao_edge_pos
add x13, x11, x4 // src_b (next) = src + sao_edge_pos
2: // process 16 bytes
ld1 {v3.16b}, [x11], #16 // load src
ld1 {v4.16b}, [x12], #16 // load src_a (prev)
ld1 {v5.16b}, [x13], #16 // load src_b (next)
subs x14, x14, #16
cmhi v16.16b, v4.16b, v3.16b // (prev > cur)
cmhi v17.16b, v3.16b, v4.16b // (cur > prev)
cmhi v18.16b, v5.16b, v3.16b // (next > cur)
cmhi v19.16b, v3.16b, v5.16b // (cur > next)
sub v20.16b, v16.16b, v17.16b // diff0 = CMP(cur, prev) = (cur > prev) - (cur < prev)
sub v21.16b, v18.16b, v19.16b // diff1 = CMP(cur, next) = (cur > next) - (cur < next)
add v20.16b, v20.16b, v21.16b // diff = diff0 + diff1
add v20.16b, v20.16b, v2.16b // offset_val = diff + 2
tbl v16.16b, {v0.16b}, v20.16b
tbl v17.16b, {v1.16b}, v20.16b
uxtl v20.8h, v3.8b // src[0:7]
uxtl2 v21.8h, v3.16b // src[7:15]
zip1 v18.16b, v16.16b, v17.16b // sao_offset_val lower ->
zip2 v19.16b, v16.16b, v17.16b // sao_offset_val upper ->
sqadd v20.8h, v18.8h, v20.8h // + sao_offset_val
sqadd v21.8h, v19.8h, v21.8h
sqxtun v3.8b, v20.8h
sqxtun2 v3.16b, v21.8h
st1 {v3.16b}, [x0], #16
// filtered 16 bytes
b.ne 2b // do we have width to filter?
// no width to filter, setup next line
subs w6, w6, #1 // filtered line
add x11, x11, x15 // stride src to next line
add x0, x0, x16 // stride dst to next line
b.ne 1b // do we have lines to process?
// no lines to filter
ret
endfunc
function ff_vvc_sao_edge_filter_8x8_8_neon, export=1
adr x7, .Lvvc_sao_edge_pos
mov x15, #VVC_SAO_STRIDE
b 1f
endfunc
// ff_hevc_sao_edge_filter_8x8_8_neon(char *dst, char *src, ptrdiff stride_dst,
// int16 *sao_offset_val, int eo, int width, int height)
function ff_hevc_sao_edge_filter_8x8_8_neon, export=1
adr x7, .Lhevc_sao_edge_pos
mov x15, #HEVC_SAO_STRIDE
1:
ldr w4, [x7, w4, uxtw #2]
ld1 {v3.8h}, [x3]
mov v3.h[7], v3.h[0]
mov v3.h[0], v3.h[1]
mov v3.h[1], v3.h[2]
mov v3.h[2], v3.h[7]
uzp2 v1.16b, v3.16b, v3.16b
uzp1 v0.16b, v3.16b, v3.16b
movi v2.16b, #2
add x16, x0, x2
lsl x2, x2, #1
mov x8, x1
sub x9, x1, x4
add x10, x1, x4
1: ld1 {v3.d}[0], [ x8], x15
ld1 {v4.d}[0], [ x9], x15
ld1 {v5.d}[0], [x10], x15
ld1 {v3.d}[1], [ x8], x15
ld1 {v4.d}[1], [ x9], x15
ld1 {v5.d}[1], [x10], x15
subs w6, w6, #2
cmhi v16.16b, v4.16b, v3.16b
cmhi v17.16b, v3.16b, v4.16b
cmhi v18.16b, v5.16b, v3.16b
cmhi v19.16b, v3.16b, v5.16b
sub v20.16b, v16.16b, v17.16b
sub v21.16b, v18.16b, v19.16b
add v20.16b, v20.16b, v21.16b
add v20.16b, v20.16b, v2.16b
tbl v16.16b, {v0.16b}, v20.16b
tbl v17.16b, {v1.16b}, v20.16b
uxtl v20.8h, v3.8b
uxtl2 v21.8h, v3.16b
zip1 v18.16b, v16.16b, v17.16b
zip2 v19.16b, v16.16b, v17.16b
sqadd v20.8h, v18.8h, v20.8h
sqadd v21.8h, v19.8h, v21.8h
sqxtun v6.8b, v20.8h
sqxtun v7.8b, v21.8h
st1 {v6.8b}, [ x0], x2
st1 {v7.8b}, [x16], x2
b.ne 1b
ret
endfunc
@@ -0,0 +1,600 @@
/* -*-arm64-*-
* vim: syntax=arm64asm
*
* Copyright (c) 2014 Seppo Tomperi <seppo.tomperi@vtt.fi>
* Copyright (c) 2023 J. Dekker <jdek@itanimul.li>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
.macro hevc_loop_filter_chroma_start bitdepth
mov x4, x30
ldr w14, [x2]
ldr w15, [x2, #4]
.if \bitdepth > 8
lsl w14, w14, #(\bitdepth - 8)
lsl w15, w15, #(\bitdepth - 8)
.endif
adds w2, w14, w15
b.eq 1f
dup v16.4h, w14
dup v17.4h, w15
trn1 v16.2d, v16.2d, v17.2d
.if \bitdepth > 8
mvni v19.8h, #((0xff << (\bitdepth - 8)) & 0xff), lsl #8
movi v18.8h, #0
.endif
neg v17.8h, v16.8h
.endm
.macro hevc_loop_filter_chroma_body bitdepth
.if \bitdepth <= 8
uxtl v20.8h, v0.8b // p1
uxtl v1.8h, v1.8b // p0
uxtl v2.8h, v2.8b // q0
uxtl v23.8h, v3.8b // q1
va .req v20
vb .req v23
.else // required to specify both cases as we are unable to do: v0 .req v20
va .req v0
vb .req v3
.endif
sub v5.8h, v2.8h, v1.8h // q0 - p0
sub v6.8h, va.8h, vb.8h // p1 - q1
shl v5.8h, v5.8h, #2
add v5.8h, v6.8h, v5.8h
srshr v5.8h, v5.8h, #3
clip v17.8h, v16.8h, v5.8h
sqadd v1.8h, v1.8h, v5.8h // p0 + delta
sqsub v2.8h, v2.8h, v5.8h // q0 - delta
.if \bitdepth <= 8
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
.else
clip v18.8h, v19.8h, v1.8h, v2.8h
.endif
.unreq va
.unreq vb
.endm
function hevc_loop_filter_chroma_body_8_neon, export=0
hevc_loop_filter_chroma_body 8
ret
endfunc
function hevc_loop_filter_chroma_body_10_neon, export=0
hevc_loop_filter_chroma_body_12_neon:
hevc_loop_filter_chroma_body 10
ret
endfunc
// void ff_hevc_h_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int *_tc, uint8_t *_no_p, uint8_t *_no_q);
.macro hevc_h_loop_filter_chroma bitdepth
function ff_hevc_h_loop_filter_chroma_\bitdepth\()_neon, export=1
hevc_loop_filter_chroma_start \bitdepth
sub x0, x0, x1, lsl #1
.if \bitdepth > 8
ld1 {v0.8h}, [x0], x1
ld1 {v1.8h}, [x0], x1
ld1 {v2.8h}, [x0], x1
ld1 {v3.8h}, [x0]
.else
ld1 {v0.8b}, [x0], x1
ld1 {v1.8b}, [x0], x1
ld1 {v2.8b}, [x0], x1
ld1 {v3.8b}, [x0]
.endif
sub x0, x0, x1, lsl #1
bl hevc_loop_filter_chroma_body_\bitdepth\()_neon
.if \bitdepth > 8
st1 {v1.8h}, [x0], x1
st1 {v2.8h}, [x0]
.else
st1 {v1.8b}, [x0], x1
st1 {v2.8b}, [x0]
.endif
1: ret x4
endfunc
.endm
.macro hevc_v_loop_filter_chroma bitdepth
function ff_hevc_v_loop_filter_chroma_\bitdepth\()_neon, export=1
hevc_loop_filter_chroma_start \bitdepth
.if \bitdepth > 8
sub x0, x0, #4
add x3, x0, x1
lsl x1, x1, #1
ld1 {v0.d}[0], [x0], x1
ld1 {v1.d}[0], [x3], x1
ld1 {v2.d}[0], [x0], x1
ld1 {v3.d}[0], [x3], x1
ld1 {v0.d}[1], [x0], x1
ld1 {v1.d}[1], [x3], x1
ld1 {v2.d}[1], [x0], x1
ld1 {v3.d}[1], [x3], x1
transpose_4x8H v0, v1, v2, v3, v28, v29, v30, v31
.else
sub x0, x0, #2
add x3, x0, x1
lsl x1, x1, #1
ld1 {v0.s}[0], [x0], x1
ld1 {v1.s}[0], [x3], x1
ld1 {v2.s}[0], [x0], x1
ld1 {v3.s}[0], [x3], x1
ld1 {v0.s}[1], [x0], x1
ld1 {v1.s}[1], [x3], x1
ld1 {v2.s}[1], [x0], x1
ld1 {v3.s}[1], [x3], x1
transpose_4x8B v0, v1, v2, v3, v28, v29, v30, v31
.endif
sub x0, x0, x1, lsl #2
sub x3, x3, x1, lsl #2
bl hevc_loop_filter_chroma_body_\bitdepth\()_neon
.if \bitdepth > 8
transpose_4x8H v0, v1, v2, v3, v28, v29, v30, v31
st1 {v0.d}[0], [x0], x1
st1 {v1.d}[0], [x3], x1
st1 {v2.d}[0], [x0], x1
st1 {v3.d}[0], [x3], x1
st1 {v0.d}[1], [x0], x1
st1 {v1.d}[1], [x3], x1
st1 {v2.d}[1], [x0], x1
st1 {v3.d}[1], [x3]
.else
transpose_4x8B v0, v1, v2, v3, v28, v29, v30, v31
st1 {v0.s}[0], [x0], x1
st1 {v1.s}[0], [x3], x1
st1 {v2.s}[0], [x0], x1
st1 {v3.s}[0], [x3], x1
st1 {v0.s}[1], [x0], x1
st1 {v1.s}[1], [x3], x1
st1 {v2.s}[1], [x0], x1
st1 {v3.s}[1], [x3]
.endif
1: ret x4
endfunc
.endm
hevc_h_loop_filter_chroma 8
hevc_h_loop_filter_chroma 10
hevc_h_loop_filter_chroma 12
hevc_v_loop_filter_chroma 8
hevc_v_loop_filter_chroma 10
hevc_v_loop_filter_chroma 12
.macro hevc_loop_filter_luma_body bitdepth
function hevc_loop_filter_luma_body_\bitdepth\()_neon, export=0
.if \bitdepth > 8
lsl w2, w2, #(\bitdepth - 8) // beta <<= BIT_DEPTH - 8
.else
uxtl v0.8h, v0.8b
uxtl v1.8h, v1.8b
uxtl v2.8h, v2.8b
uxtl v3.8h, v3.8b
uxtl v4.8h, v4.8b
uxtl v5.8h, v5.8b
uxtl v6.8h, v6.8b
uxtl v7.8h, v7.8b
.endif
ldr w7, [x3] // tc[0]
ldr w8, [x3, #4] // tc[1]
dup v18.4h, w7
dup v19.4h, w8
trn1 v18.2d, v18.2d, v19.2d
.if \bitdepth > 8
shl v18.8h, v18.8h, #(\bitdepth - 8)
.endif
dup v27.8h, w2 // beta
// tc25
shl v19.8h, v18.8h, #2 // * 4
add v19.8h, v19.8h, v18.8h // (tc * 5)
srshr v19.8h, v19.8h, #1 // (tc * 5 + 1) >> 1
sshr v17.8h, v27.8h, #2 // beta2
////// beta_2 check
// dp0 = abs(P2 - 2 * P1 + P0)
add v22.8h, v3.8h, v1.8h
shl v23.8h, v2.8h, #1
sabd v30.8h, v22.8h, v23.8h
// dq0 = abs(Q2 - 2 * Q1 + Q0)
add v21.8h, v6.8h, v4.8h
shl v26.8h, v5.8h, #1
sabd v31.8h, v21.8h, v26.8h
// d0 = dp0 + dq0
add v20.8h, v30.8h, v31.8h
shl v25.8h, v20.8h, #1
// (d0 << 1) < beta_2
cmgt v23.8h, v17.8h, v25.8h
////// beta check
// d0 + d3 < beta
mov x9, #0xFFFF00000000FFFF
dup v24.2d, x9
and v25.16b, v24.16b, v20.16b
addp v25.8h, v25.8h, v25.8h // 1+0 0+1 1+0 0+1
addp v25.4h, v25.4h, v25.4h // 1+0+0+1 1+0+0+1
cmgt v25.4h, v27.4h, v25.4h // lower/upper mask in h[0/1]
mov w9, v25.s[0]
cmp w9, #0
sxtl v26.4s, v25.4h
sxtl v16.2d, v26.2s // full skip mask
b.eq 3f // skip both blocks
// TODO: we can check the full skip mask with the weak/strong mask to
// potentially skip weak or strong calculation entirely if we only have one
////// beta_3 check
// abs(P3 - P0) + abs(Q3 - Q0) < beta_3
sshr v17.8h, v17.8h, #1 // beta_3
sabd v20.8h, v0.8h, v3.8h
saba v20.8h, v7.8h, v4.8h
cmgt v21.8h, v17.8h, v20.8h
and v23.16b, v23.16b, v21.16b
////// tc25 check
// abs(P0 - Q0) < tc25
sabd v20.8h, v3.8h, v4.8h
cmgt v21.8h, v19.8h, v20.8h
and v23.16b, v23.16b, v21.16b
////// Generate low/high line max from lines 0/3/4/7
// mask out lines 2/3/5/6
not v20.16b, v24.16b // 0x0000FFFFFFFF0000
orr v23.16b, v23.16b, v20.16b
// generate weak/strong mask
uminp v23.8h, v23.8h, v23.8h // extend to singles
sxtl v23.4s, v23.4h
uminp v26.4s, v23.4s, v23.4s // check lines
// extract to gpr
ext v25.16b, v26.16b, v26.16b, #2
zip1 v17.4s, v26.4s, v26.4s
mov w12, v25.s[0]
mov w11, #0x0000FFFF
mov w13, #0xFFFF0000
// FFFF FFFF -> strong strong
// FFFF 0000 -> strong weak
// 0000 FFFF -> weak strong
// 0000 0000 -> weak weak
cmp w12, w13
b.hi 0f // only strong/strong, skip weak nd_p/nd_q calc
////// weak nd_p/nd_q
// d0+d3
and v30.16b, v30.16b, v24.16b // d0 __ __ d3 d4 __ __ d7
and v31.16b, v31.16b, v24.16b
addp v30.8h, v30.8h, v30.8h // [d0+__ __+d3 d4+__ __+d7] [ ... ]
addp v31.8h, v31.8h, v31.8h // [d0+d3 d4+d7]
addp v30.4h, v30.4h, v30.4h
addp v31.4h, v31.4h, v31.4h
// ((beta + (beta >> 1)) >> 3)
sshr v21.8h, v27.8h, #1
add v21.8h, v21.8h, v27.8h
sshr v21.8h, v21.8h, #3
// nd_p = dp0 + dp3 < ((beta + (beta >> 1)) >> 3)
cmgt v30.8h, v21.8h, v30.8h
// nd_q = dq0 + dq3 < ((beta + (beta >> 1)) >> 3)
cmgt v31.8h, v21.8h, v31.8h
sxtl v30.4s, v30.4h
sxtl v31.4s, v31.4h
sxtl v28.2d, v30.2s
sxtl v29.2d, v31.2s
cmp w12, w11
b.lo 1f // can only be weak weak, skip strong
0: // STRONG FILTER
// P0 = p0 + av_clip(((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) - p0, -tc3, tc3);
add v21.8h, v2.8h, v3.8h // (p1 + p0
add v21.8h, v4.8h, v21.8h // + q0)
shl v21.8h, v21.8h, #1 // * 2
add v22.8h, v1.8h, v5.8h // (p2 + q1)
add v21.8h, v22.8h, v21.8h // +
srshr v21.8h, v21.8h, #3 // >> 3
sub v21.8h, v21.8h, v3.8h // - p0
// P1 = p1 + av_clip(((p2 + p1 + p0 + q0 + 2) >> 2) - p1, -tc2, tc2);
add v22.8h, v1.8h, v2.8h
add v23.8h, v3.8h, v4.8h
add v22.8h, v22.8h, v23.8h
srshr v22.8h, v22.8h, #2
sub v22.8h, v22.8h, v2.8h
// P2 = p2 + av_clip(((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) - p2, -tc, tc);
add v23.8h, v0.8h, v1.8h // p3 + p2
add v24.8h, v3.8h, v4.8h // p0 + q0
shl v23.8h, v23.8h, #1 // * 2
add v23.8h, v23.8h, v24.8h
add v24.8h, v1.8h, v2.8h // p2 + p1
add v23.8h, v23.8h, v24.8h
srshr v23.8h, v23.8h, #3
sub v23.8h, v23.8h, v1.8h
// Q0 = q0 + av_clip(((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) - q0, -tc3, tc3);
add v24.8h, v3.8h, v4.8h // (p0 + q0
add v24.8h, v5.8h, v24.8h // + q1)
shl v24.8h, v24.8h, #1 // * 2
add v25.8h, v2.8h, v6.8h // (p1 + q2)
add v24.8h, v25.8h, v24.8h // +
srshr v24.8h, v24.8h, #3 // >> 3
sub v24.8h, v24.8h, v4.8h // - q0
// Q1 = q1 + av_clip(((p0 + q0 + q1 + q2 + 2) >> 2) - q1, -tc2, tc2);
add v25.8h, v6.8h, v5.8h
add v26.8h, v3.8h, v4.8h
add v25.8h, v25.8h, v26.8h
srshr v25.8h, v25.8h, #2
sub v25.8h, v25.8h, v5.8h
// Q2 = q2 + av_clip(((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) - q2, -tc, tc);
add v26.8h, v7.8h, v6.8h
add v27.8h, v6.8h, v5.8h
shl v26.8h, v26.8h, #1
add v26.8h, v26.8h, v27.8h
add v27.8h, v3.8h, v4.8h
add v26.8h, v26.8h, v27.8h
srshr v26.8h, v26.8h, #3
sub v26.8h, v26.8h, v6.8h
// this clip should work properly
shl v30.8h, v18.8h, #1 // tc2
neg v31.8h, v30.8h // -tc2
clip v31.8h, v30.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h
and v21.16b, v21.16b, v16.16b
and v22.16b, v22.16b, v16.16b
and v23.16b, v23.16b, v16.16b
and v24.16b, v24.16b, v16.16b
and v25.16b, v25.16b, v16.16b
and v26.16b, v26.16b, v16.16b
add v23.8h, v23.8h, v1.8h // careful
add v22.8h, v22.8h, v2.8h
add v21.8h, v21.8h, v3.8h
add v24.8h, v24.8h, v4.8h
add v25.8h, v25.8h, v5.8h
add v26.8h, v26.8h, v6.8h
cmp w12, w13
b.hi 2f // only strong/strong, skip weak
1: // WEAK FILTER
// delta0 = (9 * (q0 - p0) - 3 * (q1 - p1) + 8) >> 4
.if \bitdepth < 12
sub v27.8h, v4.8h, v3.8h // q0 - p0
shl v30.8h, v27.8h, #3 // * 8
add v27.8h, v27.8h, v30.8h // 9 * (q0 - p0)
sub v30.8h, v5.8h, v2.8h // q1 - p1
shl v31.8h, v30.8h, #1 // * 2
sub v27.8h, v27.8h, v31.8h
sub v27.8h, v27.8h, v30.8h // - 3 * (q1 - p1)
srshr v27.8h, v27.8h, #4
.else
sub v19.8h, v4.8h, v3.8h // q0 - p0
sub v20.8h, v5.8h, v2.8h // q1 - p1
sshll v30.4s, v19.4h, #3 // * 8
sshll2 v31.4s, v19.8h, #3
shl v27.8h, v20.8h, #1
saddw v30.4s, v30.4s, v19.4h // 9 * (q0 - p0)
saddw2 v31.4s, v31.4s, v19.8h
saddl v19.4s, v27.4h, v20.4h // 3 * (q1 - p1)
saddl2 v20.4s, v27.8h, v20.8h
sub v19.4s, v30.4s, v19.4s
sub v20.4s, v31.4s, v20.4s
sqrshrn v27.4h, v19.4s, #4
sqrshrn2 v27.8h, v20.4s, #4
.endif
// delta0 10tc check mask
shl v30.8h, v18.8h, #1 // * 2
shl v31.8h, v18.8h, #3 // * 8
add v30.8h, v30.8h, v31.8h // 10 * tc
abs v31.8h, v27.8h
cmgt v20.8h, v30.8h, v31.8h // abs(delta0) < 10 * tc
and v20.16b, v20.16b, v16.16b // combine with full mask
neg v31.8h, v18.8h // -tc
clip v31.8h, v18.8h, v27.8h // delta0 = av_clip(delta0, -tc, tc)
// deltap1 = av_clip((((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1, -tc_2, tc_2)
add v30.8h, v1.8h, v3.8h
srshr v30.8h, v30.8h, #1
sub v30.8h, v30.8h, v2.8h
add v30.8h, v30.8h, v27.8h
sshr v30.8h, v30.8h, #1
// p3 p2 p1 p0 q0 q1 q2 q3
// v0 v1 v2 v3 v4 v5 v6 v7
// deltaq1 = av_clip((((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1, -tc_2, tc_2);
add v31.8h, v6.8h, v4.8h
srshr v31.8h, v31.8h, #1
sub v31.8h, v31.8h, v5.8h
sub v31.8h, v31.8h, v27.8h
sshr v31.8h, v31.8h, #1
// apply nd_p nd_q mask to deltap1/deltaq1
and v30.16b, v30.16b, v28.16b
and v31.16b, v31.16b, v29.16b
// apply full skip mask to deltap1/deltaq1/delta0
and v30.16b, v30.16b, v20.16b
and v27.16b, v27.16b, v20.16b
and v31.16b, v31.16b, v20.16b
// clip P1/Q1 to -tc_2, tc_2
sshr v18.8h, v18.8h, #1 // tc2
neg v28.8h, v18.8h
clip v28.8h, v18.8h, v30.8h, v31.8h
// P0 = av_clip_pixel(p0 + delta0)
// Q0 = av_clip_pixel(q0 - delta0)
add v29.8h, v3.8h, v27.8h // P0
sub v27.8h, v4.8h, v27.8h // Q0
// P1 = av_clip_pixel(p1 + deltap1)
// Q1 = av_clip_pixel(q1 + deltaq1)
add v30.8h, v2.8h, v30.8h // P1
add v31.8h, v5.8h, v31.8h // Q1
2: // MIX WEAK/STRONG
mov v19.16b, v1.16b
mov v20.16b, v6.16b
// copy selection mask
mov v1.16b, v17.16b
mov v2.16b, v17.16b
mov v3.16b, v17.16b
mov v4.16b, v17.16b
mov v5.16b, v17.16b
mov v6.16b, v17.16b
// select
bsl v1.16b, v23.16b, v19.16b // P2 strong/orig
bsl v2.16b, v22.16b, v30.16b // P1 strong/weak
bsl v3.16b, v21.16b, v29.16b // P0 strong/weak
bsl v4.16b, v24.16b, v27.16b // Q0 strong/weak
bsl v5.16b, v25.16b, v31.16b // Q1 strong/weak
bsl v6.16b, v26.16b, v20.16b // Q2 strong/orig
// NOTE: Q3/P3 are unchanged
.if \bitdepth > 8
movi v19.8h, #0
dup v20.8h, w14
clip v19.8h, v20.8h, v1.8h, v2.8h, v3.8h, v4.8h, v5.8h, v6.8h
.else
sqxtun v0.8b, v0.8h
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
sqxtun v3.8b, v3.8h
sqxtun v4.8b, v4.8h
sqxtun v5.8b, v5.8h
sqxtun v6.8b, v6.8h
sqxtun v7.8b, v7.8h
.endif
ret
3: ret x6
endfunc
.endm
hevc_loop_filter_luma_body 8
hevc_loop_filter_luma_body 10
hevc_loop_filter_luma_body 12
// hevc_v_loop_filter_luma(uint8_t *pix, ptrdiff_t stride, int beta, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q)
.macro hevc_loop_filter_luma dir, bitdepth
function ff_hevc_\dir\()_loop_filter_luma_\bitdepth\()_neon, export=1
mov x6, x30
.ifc \dir, v
.if \bitdepth > 8
sub x0, x0, #8
.else
sub x0, x0, #4
.endif
.else
sub x0, x0, x1, lsl #2 // -4 * xstride
.endif
mov x10, x0
.if \bitdepth > 8
ld1 {v0.8h}, [x0], x1
ld1 {v1.8h}, [x0], x1
ld1 {v2.8h}, [x0], x1
ld1 {v3.8h}, [x0], x1
ld1 {v4.8h}, [x0], x1
ld1 {v5.8h}, [x0], x1
ld1 {v6.8h}, [x0], x1
ld1 {v7.8h}, [x0]
mov w14, #((1 << \bitdepth) - 1)
.ifc \dir, v
transpose_8x8H v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
.else
ld1 {v0.8b}, [x0], x1
ld1 {v1.8b}, [x0], x1
ld1 {v2.8b}, [x0], x1
ld1 {v3.8b}, [x0], x1
ld1 {v4.8b}, [x0], x1
ld1 {v5.8b}, [x0], x1
ld1 {v6.8b}, [x0], x1
ld1 {v7.8b}, [x0]
.ifc \dir, v
transpose_8x8B v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
.endif
bl hevc_loop_filter_luma_body_\bitdepth\()_neon
.if \bitdepth > 8
.ifc \dir, v
transpose_8x8H v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
st1 {v0.8h}, [x10], x1
st1 {v1.8h}, [x10], x1
st1 {v2.8h}, [x10], x1
st1 {v3.8h}, [x10], x1
st1 {v4.8h}, [x10], x1
st1 {v5.8h}, [x10], x1
st1 {v6.8h}, [x10], x1
st1 {v7.8h}, [x10]
.else
.ifc \dir, v
transpose_8x8B v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
st1 {v0.8b}, [x10], x1
st1 {v1.8b}, [x10], x1
st1 {v2.8b}, [x10], x1
st1 {v3.8b}, [x10], x1
st1 {v4.8b}, [x10], x1
st1 {v5.8b}, [x10], x1
st1 {v6.8b}, [x10], x1
st1 {v7.8b}, [x10]
.endif
ret x6
endfunc
.endm
hevc_loop_filter_luma h, 8
hevc_loop_filter_luma h, 10
hevc_loop_filter_luma h, 12
hevc_loop_filter_luma v, 8
hevc_loop_filter_luma v, 10
hevc_loop_filter_luma v, 12
@@ -0,0 +1,938 @@
/*
* ARM NEON optimised IDCT functions for HEVC decoding
* Copyright (c) 2014 Seppo Tomperi <seppo.tomperi@vtt.fi>
* Copyright (c) 2017 Alexandra Hájková
*
* Ported from arm/hevcdsp_idct_neon.S by
* Copyright (c) 2020 Reimar Döffinger
* Copyright (c) 2023 J. Dekker <jdek@itanimul.li>
* Copyright (c) 2023 xu fulong <839789740@qq.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
const trans, align=4
.short 64, 83, 64, 36
.short 89, 75, 50, 18
.short 90, 87, 80, 70
.short 57, 43, 25, 9
.short 90, 90, 88, 85
.short 82, 78, 73, 67
.short 61, 54, 46, 38
.short 31, 22, 13, 4
endconst
function ff_hevc_add_residual_4x4_8_neon, export=1
ld1 {v0.8h-v1.8h}, [x1]
ld1 {v2.s}[0], [x0], x2
ld1 {v2.s}[1], [x0], x2
ld1 {v2.s}[2], [x0], x2
ld1 {v2.s}[3], [x0], x2
sub x0, x0, x2, lsl #2
uxtl v6.8h, v2.8b
uxtl2 v7.8h, v2.16b
sqadd v0.8h, v0.8h, v6.8h
sqadd v1.8h, v1.8h, v7.8h
sqxtun v0.8b, v0.8h
sqxtun2 v0.16b, v1.8h
st1 {v0.s}[0], [x0], x2
st1 {v0.s}[1], [x0], x2
st1 {v0.s}[2], [x0], x2
st1 {v0.s}[3], [x0], x2
ret
endfunc
function ff_hevc_add_residual_8x8_8_neon, export=1
add x12, x0, x2
add x2, x2, x2
mov x3, #8
1: subs x3, x3, #2
ld1 {v2.d}[0], [x0]
ld1 {v2.d}[1], [x12]
uxtl v3.8h, v2.8b
ld1 {v0.8h-v1.8h}, [x1], #32
uxtl2 v2.8h, v2.16b
sqadd v0.8h, v0.8h, v3.8h
sqadd v1.8h, v1.8h, v2.8h
sqxtun v0.8b, v0.8h
sqxtun2 v0.16b, v1.8h
st1 {v0.d}[0], [x0], x2
st1 {v0.d}[1], [x12], x2
bne 1b
ret
endfunc
function ff_hevc_add_residual_16x16_8_neon, export=1
mov x3, #16
add x12, x0, x2
add x2, x2, x2
1: subs x3, x3, #2
ld1 {v16.16b}, [x0]
ld1 {v0.8h-v3.8h}, [x1], #64
ld1 {v19.16b}, [x12]
uxtl v17.8h, v16.8b
uxtl2 v18.8h, v16.16b
uxtl v20.8h, v19.8b
uxtl2 v21.8h, v19.16b
sqadd v0.8h, v0.8h, v17.8h
sqadd v1.8h, v1.8h, v18.8h
sqadd v2.8h, v2.8h, v20.8h
sqadd v3.8h, v3.8h, v21.8h
sqxtun v0.8b, v0.8h
sqxtun2 v0.16b, v1.8h
sqxtun v1.8b, v2.8h
sqxtun2 v1.16b, v3.8h
st1 {v0.16b}, [x0], x2
st1 {v1.16b}, [x12], x2
bne 1b
ret
endfunc
function ff_hevc_add_residual_32x32_8_neon, export=1
add x12, x0, x2
add x2, x2, x2
mov x3, #32
1: subs x3, x3, #2
ld1 {v20.16b, v21.16b}, [x0]
uxtl v16.8h, v20.8b
uxtl2 v17.8h, v20.16b
ld1 {v22.16b, v23.16b}, [x12]
uxtl v18.8h, v21.8b
uxtl2 v19.8h, v21.16b
uxtl v20.8h, v22.8b
ld1 {v0.8h-v3.8h}, [x1], #64
ld1 {v4.8h-v7.8h}, [x1], #64
uxtl2 v21.8h, v22.16b
uxtl v22.8h, v23.8b
uxtl2 v23.8h, v23.16b
sqadd v0.8h, v0.8h, v16.8h
sqadd v1.8h, v1.8h, v17.8h
sqadd v2.8h, v2.8h, v18.8h
sqadd v3.8h, v3.8h, v19.8h
sqadd v4.8h, v4.8h, v20.8h
sqadd v5.8h, v5.8h, v21.8h
sqadd v6.8h, v6.8h, v22.8h
sqadd v7.8h, v7.8h, v23.8h
sqxtun v0.8b, v0.8h
sqxtun2 v0.16b, v1.8h
sqxtun v1.8b, v2.8h
sqxtun2 v1.16b, v3.8h
sqxtun v2.8b, v4.8h
sqxtun2 v2.16b, v5.8h
st1 {v0.16b, v1.16b}, [x0], x2
sqxtun v3.8b, v6.8h
sqxtun2 v3.16b, v7.8h
st1 {v2.16b, v3.16b}, [x12], x2
bne 1b
ret
endfunc
.macro add_res bitdepth
function ff_hevc_add_residual_4x4_\bitdepth\()_neon, export=1
mvni v21.8h, #((0xFF << (\bitdepth - 8)) & 0xFF), lsl #8
b hevc_add_residual_4x4_16_neon
endfunc
function ff_hevc_add_residual_8x8_\bitdepth\()_neon, export=1
mvni v21.8h, #((0xFF << (\bitdepth - 8)) & 0xFF), lsl #8
b hevc_add_residual_8x8_16_neon
endfunc
function ff_hevc_add_residual_16x16_\bitdepth\()_neon, export=1
mvni v21.8h, #((0xFF << (\bitdepth - 8)) & 0xFF), lsl #8
b hevc_add_residual_16x16_16_neon
endfunc
function ff_hevc_add_residual_32x32_\bitdepth\()_neon, export=1
mvni v21.8h, #((0xFF << (\bitdepth - 8)) & 0xFF), lsl #8
b hevc_add_residual_32x32_16_neon
endfunc
.endm
add_res 10
add_res 12
function hevc_add_residual_4x4_16_neon, export=0
mov x12, x0
ld1 {v0.8h-v1.8h}, [x1]
ld1 {v2.d}[0], [x12], x2
ld1 {v2.d}[1], [x12], x2
ld1 {v3.d}[0], [x12], x2
sqadd v0.8h, v0.8h, v2.8h
ld1 {v3.d}[1], [x12], x2
movi v4.8h, #0
sqadd v1.8h, v1.8h, v3.8h
clip v4.8h, v21.8h, v0.8h, v1.8h
st1 {v0.d}[0], [x0], x2
st1 {v0.d}[1], [x0], x2
st1 {v1.d}[0], [x0], x2
st1 {v1.d}[1], [x0], x2
ret
endfunc
function hevc_add_residual_8x8_16_neon, export=0
add x12, x0, x2
add x2, x2, x2
mov x3, #8
movi v4.8h, #0
1: subs x3, x3, #2
ld1 {v0.8h-v1.8h}, [x1], #32
ld1 {v2.8h}, [x0]
sqadd v0.8h, v0.8h, v2.8h
ld1 {v3.8h}, [x12]
sqadd v1.8h, v1.8h, v3.8h
clip v4.8h, v21.8h, v0.8h, v1.8h
st1 {v0.8h}, [x0], x2
st1 {v1.8h}, [x12], x2
bne 1b
ret
endfunc
function hevc_add_residual_16x16_16_neon, export=0
mov x3, #16
movi v20.8h, #0
add x12, x0, x2
add x2, x2, x2
1: subs x3, x3, #2
ld1 {v16.8h-v17.8h}, [x0]
ld1 {v0.8h-v3.8h}, [x1], #64
sqadd v0.8h, v0.8h, v16.8h
ld1 {v18.8h-v19.8h}, [x12]
sqadd v1.8h, v1.8h, v17.8h
sqadd v2.8h, v2.8h, v18.8h
sqadd v3.8h, v3.8h, v19.8h
clip v20.8h, v21.8h, v0.8h, v1.8h, v2.8h, v3.8h
st1 {v0.8h-v1.8h}, [x0], x2
st1 {v2.8h-v3.8h}, [x12], x2
bne 1b
ret
endfunc
function hevc_add_residual_32x32_16_neon, export=0
mov x3, #32
movi v20.8h, #0
1: subs x3, x3, #1
ld1 {v0.8h -v3.8h}, [x1], #64
ld1 {v16.8h-v19.8h}, [x0]
sqadd v0.8h, v0.8h, v16.8h
sqadd v1.8h, v1.8h, v17.8h
sqadd v2.8h, v2.8h, v18.8h
sqadd v3.8h, v3.8h, v19.8h
clip v20.8h, v21.8h, v0.8h, v1.8h, v2.8h, v3.8h
st1 {v0.8h-v3.8h}, [x0], x2
bne 1b
ret
endfunc
.macro tr_4x4 in0, in1, in2, in3, out0, out1, out2, out3, shift
sshll v20.4s, \in0, #6
sshll v21.4s, \in0, #6
smull v22.4s, \in1, v4.h[1]
smull v23.4s, \in1, v4.h[3]
smlal v20.4s, \in2, v4.h[0] //e0
smlsl v21.4s, \in2, v4.h[0] //e1
smlal v22.4s, \in3, v4.h[3] //o0
smlsl v23.4s, \in3, v4.h[1] //o1
add v24.4s, v20.4s, v22.4s
sub v20.4s, v20.4s, v22.4s
add v22.4s, v21.4s, v23.4s
sub v21.4s, v21.4s, v23.4s
sqrshrn \out0, v24.4s, #\shift
sqrshrn \out3, v20.4s, #\shift
sqrshrn \out1, v22.4s, #\shift
sqrshrn \out2, v21.4s, #\shift
.endm
.macro idct_4x4 bitdepth
function ff_hevc_idct_4x4_\bitdepth\()_neon, export=1
ld1 {v0.4h-v3.4h}, [x0]
movrel x1, trans
ld1 {v4.4h}, [x1]
tr_4x4 v0.4h, v1.4h, v2.4h, v3.4h, v16.4h, v17.4h, v18.4h, v19.4h, 7
transpose_4x8H v16, v17, v18, v19, v26, v27, v28, v29
tr_4x4 v16.4h, v17.4h, v18.4h, v19.4h, v0.4h, v1.4h, v2.4h, v3.4h, 20 - \bitdepth
transpose_4x8H v0, v1, v2, v3, v26, v27, v28, v29
st1 {v0.4h-v3.4h}, [x0]
ret
endfunc
.endm
.macro sum_sub out, in, c, op, p
.ifc \op, +
smlal\p \out, \in, \c
.else
smlsl\p \out, \in, \c
.endif
.endm
.macro fixsqrshrn d, dt, n, m
.ifc \dt, .8h
sqrshrn2 \d\dt, \n\().4s, \m
.else
sqrshrn \n\().4h, \n\().4s, \m
mov \d\().d[0], \n\().d[0]
.endif
.endm
// uses and clobbers v28-v31 as temp registers
.macro tr_4x4_8 in0, in1, in2, in3, out0, out1, out2, out3, p1, p2
sshll\p1 v28.4s, \in0, #6
mov v29.16b, v28.16b
smull\p1 v30.4s, \in1, v0.h[1]
smull\p1 v31.4s, \in1, v0.h[3]
smlal\p2 v28.4s, \in2, v0.h[0] //e0
smlsl\p2 v29.4s, \in2, v0.h[0] //e1
smlal\p2 v30.4s, \in3, v0.h[3] //o0
smlsl\p2 v31.4s, \in3, v0.h[1] //o1
add \out0, v28.4s, v30.4s
add \out1, v29.4s, v31.4s
sub \out2, v29.4s, v31.4s
sub \out3, v28.4s, v30.4s
.endm
.macro transpose_8x8 r0, r1, r2, r3, r4, r5, r6, r7
transpose_4x8H \r0, \r1, \r2, \r3, v2, v3, v4, v5
transpose_4x8H \r4, \r5, \r6, \r7, v2, v3, v4, v5
.endm
.macro tr_8x4 shift, in0,in0t, in1,in1t, in2,in2t, in3,in3t, in4,in4t, in5,in5t, in6,in6t, in7,in7t, p1, p2
tr_4x4_8 \in0\in0t, \in2\in2t, \in4\in4t, \in6\in6t, v24.4s, v25.4s, v26.4s, v27.4s, \p1, \p2
smull\p1 v30.4s, \in1\in1t, v0.h[6]
smull\p1 v28.4s, \in1\in1t, v0.h[4]
smull\p1 v29.4s, \in1\in1t, v0.h[5]
sum_sub v30.4s, \in3\in3t, v0.h[4], -, \p1
sum_sub v28.4s, \in3\in3t, v0.h[5], +, \p1
sum_sub v29.4s, \in3\in3t, v0.h[7], -, \p1
sum_sub v30.4s, \in5\in5t, v0.h[7], +, \p2
sum_sub v28.4s, \in5\in5t, v0.h[6], +, \p2
sum_sub v29.4s, \in5\in5t, v0.h[4], -, \p2
sum_sub v30.4s, \in7\in7t, v0.h[5], +, \p2
sum_sub v28.4s, \in7\in7t, v0.h[7], +, \p2
sum_sub v29.4s, \in7\in7t, v0.h[6], -, \p2
add v31.4s, v26.4s, v30.4s
sub v26.4s, v26.4s, v30.4s
fixsqrshrn \in2,\in2t, v31, \shift
smull\p1 v31.4s, \in1\in1t, v0.h[7]
sum_sub v31.4s, \in3\in3t, v0.h[6], -, \p1
sum_sub v31.4s, \in5\in5t, v0.h[5], +, \p2
sum_sub v31.4s, \in7\in7t, v0.h[4], -, \p2
fixsqrshrn \in5,\in5t, v26, \shift
add v26.4s, v24.4s, v28.4s
sub v24.4s, v24.4s, v28.4s
add v28.4s, v25.4s, v29.4s
sub v25.4s, v25.4s, v29.4s
add v30.4s, v27.4s, v31.4s
sub v27.4s, v27.4s, v31.4s
fixsqrshrn \in0,\in0t, v26, \shift
fixsqrshrn \in7,\in7t, v24, \shift
fixsqrshrn \in1,\in1t, v28, \shift
fixsqrshrn \in6,\in6t, v25, \shift
fixsqrshrn \in3,\in3t, v30, \shift
fixsqrshrn \in4,\in4t, v27, \shift
.endm
.macro idct_8x8 bitdepth
function ff_hevc_idct_8x8_\bitdepth\()_neon, export=1
//x0 - coeffs
mov x1, x0
ld1 {v16.8h-v19.8h}, [x1], #64
ld1 {v20.8h-v23.8h}, [x1]
movrel x1, trans
ld1 {v0.8h}, [x1]
tr_8x4 7, v16,.4h, v17,.4h, v18,.4h, v19,.4h, v20,.4h, v21,.4h, v22,.4h, v23,.4h
tr_8x4 7, v16,.8h, v17,.8h, v18,.8h, v19,.8h, v20,.8h, v21,.8h, v22,.8h, v23,.8h, 2, 2
transpose_8x8 v16, v17, v18, v19, v20, v21, v22, v23
tr_8x4 20 - \bitdepth, v16,.4h, v17,.4h, v18,.4h, v19,.4h, v16,.8h, v17,.8h, v18,.8h, v19,.8h, , 2
tr_8x4 20 - \bitdepth, v20,.4h, v21,.4h, v22,.4h, v23,.4h, v20,.8h, v21,.8h, v22,.8h, v23,.8h, , 2
transpose_8x8 v16, v17, v18, v19, v20, v21, v22, v23
mov x1, x0
st1 {v16.8h-v19.8h}, [x1], #64
st1 {v20.8h-v23.8h}, [x1]
ret
endfunc
.endm
.macro butterfly e, o, tmp_p, tmp_m
add \tmp_p, \e, \o
sub \tmp_m, \e, \o
.endm
.macro tr16_8x4 in0, in1, in2, in3, offset
tr_4x4_8 \in0\().4h, \in1\().4h, \in2\().4h, \in3\().4h, v24.4s, v25.4s, v26.4s, v27.4s
smull2 v28.4s, \in0\().8h, v0.h[4]
smull2 v29.4s, \in0\().8h, v0.h[5]
smull2 v30.4s, \in0\().8h, v0.h[6]
smull2 v31.4s, \in0\().8h, v0.h[7]
sum_sub v28.4s, \in1\().8h, v0.h[5], +, 2
sum_sub v29.4s, \in1\().8h, v0.h[7], -, 2
sum_sub v30.4s, \in1\().8h, v0.h[4], -, 2
sum_sub v31.4s, \in1\().8h, v0.h[6], -, 2
sum_sub v28.4s, \in2\().8h, v0.h[6], +, 2
sum_sub v29.4s, \in2\().8h, v0.h[4], -, 2
sum_sub v30.4s, \in2\().8h, v0.h[7], +, 2
sum_sub v31.4s, \in2\().8h, v0.h[5], +, 2
sum_sub v28.4s, \in3\().8h, v0.h[7], +, 2
sum_sub v29.4s, \in3\().8h, v0.h[6], -, 2
sum_sub v30.4s, \in3\().8h, v0.h[5], +, 2
sum_sub v31.4s, \in3\().8h, v0.h[4], -, 2
butterfly v24.4s, v28.4s, v16.4s, v23.4s
butterfly v25.4s, v29.4s, v17.4s, v22.4s
butterfly v26.4s, v30.4s, v18.4s, v21.4s
butterfly v27.4s, v31.4s, v19.4s, v20.4s
add x4, sp, #\offset
st1 {v16.4s-v19.4s}, [x4], #64
st1 {v20.4s-v23.4s}, [x4]
.endm
.macro load16 in0, in1, in2, in3
ld1 {\in0}[0], [x1], x2
ld1 {\in0}[1], [x3], x2
ld1 {\in1}[0], [x1], x2
ld1 {\in1}[1], [x3], x2
ld1 {\in2}[0], [x1], x2
ld1 {\in2}[1], [x3], x2
ld1 {\in3}[0], [x1], x2
ld1 {\in3}[1], [x3], x2
.endm
.macro add_member in, t0, t1, t2, t3, t4, t5, t6, t7, op0, op1, op2, op3, op4, op5, op6, op7, p
sum_sub v21.4s, \in, \t0, \op0, \p
sum_sub v22.4s, \in, \t1, \op1, \p
sum_sub v23.4s, \in, \t2, \op2, \p
sum_sub v24.4s, \in, \t3, \op3, \p
sum_sub v25.4s, \in, \t4, \op4, \p
sum_sub v26.4s, \in, \t5, \op5, \p
sum_sub v27.4s, \in, \t6, \op6, \p
sum_sub v28.4s, \in, \t7, \op7, \p
.endm
.macro butterfly16 in0, in1, in2, in3, in4, in5, in6, in7
add v20.4s, \in0, \in1
sub \in0, \in0, \in1
add \in1, \in2, \in3
sub \in2, \in2, \in3
add \in3, \in4, \in5
sub \in4, \in4, \in5
add \in5, \in6, \in7
sub \in6, \in6, \in7
.endm
.macro store16 in0, in1, in2, in3, rx
st1 {\in0}[0], [x1], x2
st1 {\in0}[1], [x3], \rx
st1 {\in1}[0], [x1], x2
st1 {\in1}[1], [x3], \rx
st1 {\in2}[0], [x1], x2
st1 {\in2}[1], [x3], \rx
st1 {\in3}[0], [x1], x2
st1 {\in3}[1], [x3], \rx
.endm
.macro scale out0, out1, out2, out3, in0, in1, in2, in3, in4, in5, in6, in7, shift
sqrshrn \out0\().4h, \in0, \shift
sqrshrn2 \out0\().8h, \in1, \shift
sqrshrn \out1\().4h, \in2, \shift
sqrshrn2 \out1\().8h, \in3, \shift
sqrshrn \out2\().4h, \in4, \shift
sqrshrn2 \out2\().8h, \in5, \shift
sqrshrn \out3\().4h, \in6, \shift
sqrshrn2 \out3\().8h, \in7, \shift
.endm
// use temp register to transpose, then we can reuse it
.macro transpose16_4x4_2 r0, r1, r2, r3, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5
// lower halves
trn1 \tmp0\().4h, \r0\().4h, \r1\().4h
trn2 \tmp1\().4h, \r0\().4h, \r1\().4h
trn1 \tmp2\().4h, \r2\().4h, \r3\().4h
trn2 \tmp3\().4h, \r2\().4h, \r3\().4h
trn1 \tmp4\().2s, \tmp0\().2s, \tmp2\().2s
trn2 \tmp5\().2s, \tmp0\().2s, \tmp2\().2s
trn1 \tmp0\().2s, \tmp1\().2s, \tmp3\().2s
trn2 \tmp2\().2s, \tmp1\().2s, \tmp3\().2s
mov \r0\().d[0], \tmp4\().d[0]
mov \r2\().d[0], \tmp5\().d[0]
mov \r1\().d[0], \tmp0\().d[0]
mov \r3\().d[0], \tmp2\().d[0]
// upper halves in reverse order
trn1 \tmp0\().8h, \r3\().8h, \r2\().8h
trn2 \tmp1\().8h, \r3\().8h, \r2\().8h
trn1 \tmp2\().8h, \r1\().8h, \r0\().8h
trn2 \tmp3\().8h, \r1\().8h, \r0\().8h
trn1 \tmp4\().4s, \tmp0\().4s, \tmp2\().4s
trn2 \tmp5\().4s, \tmp0\().4s, \tmp2\().4s
trn1 \tmp0\().4s, \tmp1\().4s, \tmp3\().4s
trn2 \tmp2\().4s, \tmp1\().4s, \tmp3\().4s
mov \r3\().d[1], \tmp4\().d[1]
mov \r1\().d[1], \tmp5\().d[1]
mov \r2\().d[1], \tmp0\().d[1]
mov \r0\().d[1], \tmp2\().d[1]
.endm
// stores in0, in2, in4, in6 ascending from off1 and
// stores in1, in3, in5, in7 descending from off2
.macro store_to_stack off1, off2, in0, in2, in4, in6, in7, in5, in3, in1
add x1, sp, #\off1
add x3, sp, #\off2
mov x2, #-16
mov x4, #16
st1 {\in0}, [x1], x4
st1 {\in1}, [x3], x2
st1 {\in2}, [x1], x4
st1 {\in3}, [x3], x2
st1 {\in4}, [x1], x4
st1 {\in5}, [x3], x2
st1 {\in6}, [x1]
st1 {\in7}, [x3]
.endm
.macro tr_16x4 name, shift, offset, step
function func_tr_16x4_\name
mov x1, x5
add x3, x5, #(\step * 64)
mov x2, #(\step * 128)
load16 v16.d, v17.d, v18.d, v19.d
movrel x1, trans
ld1 {v0.8h}, [x1]
tr16_8x4 v16, v17, v18, v19, \offset
add x1, x5, #(\step * 32)
add x3, x5, #(\step * 3 *32)
mov x2, #(\step * 128)
load16 v20.d, v17.d, v18.d, v19.d
movrel x1, trans, 16
ld1 {v1.8h}, [x1]
smull v21.4s, v20.4h, v1.h[0]
smull v22.4s, v20.4h, v1.h[1]
smull v23.4s, v20.4h, v1.h[2]
smull v24.4s, v20.4h, v1.h[3]
smull v25.4s, v20.4h, v1.h[4]
smull v26.4s, v20.4h, v1.h[5]
smull v27.4s, v20.4h, v1.h[6]
smull v28.4s, v20.4h, v1.h[7]
add_member v20.8h, v1.h[1], v1.h[4], v1.h[7], v1.h[5], v1.h[2], v1.h[0], v1.h[3], v1.h[6], +, +, +, -, -, -, -, -, 2
add_member v17.4h, v1.h[2], v1.h[7], v1.h[3], v1.h[1], v1.h[6], v1.h[4], v1.h[0], v1.h[5], +, +, -, -, -, +, +, +
add_member v17.8h, v1.h[3], v1.h[5], v1.h[1], v1.h[7], v1.h[0], v1.h[6], v1.h[2], v1.h[4], +, -, -, +, +, +, -, -, 2
add_member v18.4h, v1.h[4], v1.h[2], v1.h[6], v1.h[0], v1.h[7], v1.h[1], v1.h[5], v1.h[3], +, -, -, +, -, -, +, +
add_member v18.8h, v1.h[5], v1.h[0], v1.h[4], v1.h[6], v1.h[1], v1.h[3], v1.h[7], v1.h[2], +, -, +, +, -, +, +, -, 2
add_member v19.4h, v1.h[6], v1.h[3], v1.h[0], v1.h[2], v1.h[5], v1.h[7], v1.h[4], v1.h[1], +, -, +, -, +, +, -, +
add_member v19.8h, v1.h[7], v1.h[6], v1.h[5], v1.h[4], v1.h[3], v1.h[2], v1.h[1], v1.h[0], +, -, +, -, +, -, +, -, 2
add x4, sp, #\offset
ld1 {v16.4s-v19.4s}, [x4], #64
butterfly16 v16.4s, v21.4s, v17.4s, v22.4s, v18.4s, v23.4s, v19.4s, v24.4s
.if \shift > 0
scale v29, v30, v31, v24, v20.4s, v16.4s, v21.4s, v17.4s, v22.4s, v18.4s, v23.4s, v19.4s, \shift
transpose16_4x4_2 v29, v30, v31, v24, v2, v3, v4, v5, v6, v7
mov x1, x6
add x3, x6, #(24 +3*32)
mov x2, #32
mov x4, #-32
store16 v29.d, v30.d, v31.d, v24.d, x4
.else
store_to_stack \offset, (\offset + 240), v20.4s, v21.4s, v22.4s, v23.4s, v19.4s, v18.4s, v17.4s, v16.4s
.endif
add x4, sp, #(\offset + 64)
ld1 {v16.4s-v19.4s}, [x4]
butterfly16 v16.4s, v25.4s, v17.4s, v26.4s, v18.4s, v27.4s, v19.4s, v28.4s
.if \shift > 0
scale v29, v30, v31, v20, v20.4s, v16.4s, v25.4s, v17.4s, v26.4s, v18.4s, v27.4s, v19.4s, \shift
transpose16_4x4_2 v29, v30, v31, v20, v2, v3, v4, v5, v6, v7
add x1, x6, #8
add x3, x6, #(16 + 3 * 32)
mov x2, #32
mov x4, #-32
store16 v29.d, v30.d, v31.d, v20.d, x4
.else
store_to_stack (\offset + 64), (\offset + 176), v20.4s, v25.4s, v26.4s, v27.4s, v19.4s, v18.4s, v17.4s, v16.4s
.endif
ret
endfunc
.endm
.macro idct_16x16 bitdepth
function ff_hevc_idct_16x16_\bitdepth\()_neon, export=1
//r0 - coeffs
mov x15, x30
// allocate a temp buffer
sub sp, sp, #640
.irp i, 0, 1, 2, 3
add x5, x0, #(8 * \i)
add x6, sp, #(8 * \i * 16)
bl func_tr_16x4_firstpass
.endr
.irp i, 0, 1, 2, 3
add x5, sp, #(8 * \i)
add x6, x0, #(8 * \i * 16)
bl func_tr_16x4_secondpass_\bitdepth
.endr
add sp, sp, #640
ret x15
endfunc
.endm
.macro load32
add x1, x5, #64
add x3, x1, #128
mov x2, #256
ld1 {v4.d}[0], [x1], x2
ld1 {v4.d}[1], [x3], x2
ld1 {v5.d}[0], [x1], x2
ld1 {v5.d}[1], [x3], x2
ld1 {v6.d}[0], [x1], x2
ld1 {v6.d}[1], [x3], x2
ld1 {v7.d}[0], [x1], x2
ld1 {v7.d}[1], [x3], x2
ld1 {v16.d}[0], [x1], x2
ld1 {v16.d}[1], [x3], x2
ld1 {v17.d}[0], [x1], x2
ld1 {v17.d}[1], [x3], x2
ld1 {v18.d}[0], [x1], x2
ld1 {v18.d}[1], [x3], x2
ld1 {v19.d}[0], [x1], x2
ld1 {v19.d}[1], [x3], x2
.endm
.macro add_member32 in, t0, t1, t2, t3, op0, op1, op2, op3, p
sum_sub v24.4s, \in, \t0, \op0, \p
sum_sub v25.4s, \in, \t1, \op1, \p
sum_sub v26.4s, \in, \t2, \op2, \p
sum_sub v27.4s, \in, \t3, \op3, \p
.endm
.macro butterfly32 in0, in1, in2, in3, out
add \out, \in0, \in1
sub \in0, \in0, \in1
add \in1, \in2, \in3
sub \in2, \in2, \in3
.endm
.macro multiply in
smull v24.4s, v4.4h, \in\().h[0]
smull v25.4s, v4.4h, \in\().h[1]
smull v26.4s, v4.4h, \in\().h[2]
smull v27.4s, v4.4h, \in\().h[3]
.endm
.macro scale_store shift
ld1 {v28.8h-v31.8h}, [x4], #64
butterfly32 v28.4s, v24.4s, v29.4s, v25.4s, v2.4s
butterfly32 v30.4s, v26.4s, v31.4s, v27.4s, v3.4s
scale v20, v21, v22, v23, v2.4s, v28.4s, v24.4s, v29.4s, v3.4s, v30.4s, v26.4s, v31.4s, \shift
transpose16_4x4_2 v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
store16 v20.d, v21.d, v22.d, v23.d, x8
// reload coefficients
ld1 {v2.4h-v3.4h}, [x9]
.endm
function tr_block1
multiply v0
add_member32 v4.8h, v0.h[1], v1.h[0], v1.h[3], v2.h[2], +, +, +, +, 2
add_member32 v5.4h, v0.h[2], v1.h[3], v3.h[0], v3.h[2], +, +, +, -
add_member32 v5.8h, v0.h[3], v2.h[2], v3.h[2], v1.h[3], +, +, -, -, 2
add_member32 v6.4h, v1.h[0], v3.h[1], v2.h[1], v0.h[0], +, +, -, -
add_member32 v6.8h, v1.h[1], v3.h[3], v1.h[0], v1.h[2], +, -, -, -, 2
add_member32 v7.4h, v1.h[2], v3.h[0], v0.h[0], v3.h[1], +, -, -, -
add_member32 v7.8h, v1.h[3], v2.h[1], v1.h[1], v2.h[3], +, -, -, +, 2
add_member32 v16.4h, v2.h[0], v1.h[2], v2.h[2], v1.h[0], +, -, -, +
add_member32 v16.8h, v2.h[1], v0.h[3], v3.h[3], v0.h[2], +, -, -, +, 2
add_member32 v17.4h, v2.h[2], v0.h[1], v2.h[3], v2.h[1], +, -, +, +
add_member32 v17.8h, v2.h[3], v0.h[2], v1.h[2], v3.h[3], +, -, +, -, 2
add_member32 v18.4h, v3.h[0], v1.h[1], v0.h[1], v2.h[0], +, -, +, -
add_member32 v18.8h, v3.h[1], v2.h[0], v0.h[3], v0.h[1], +, -, +, -, 2
add_member32 v19.4h, v3.h[2], v2.h[3], v2.h[0], v1.h[1], +, -, +, -
add_member32 v19.8h, v3.h[3], v3.h[2], v3.h[1], v3.h[0], +, -, +, -, 2
ret
endfunc
function tr_block2
multiply v1
add_member32 v4.8h, v3.h[1], v3.h[3], v3.h[0], v2.h[1], +, -, -, -, 2
add_member32 v5.4h, v2.h[1], v1.h[0], v0.h[0], v1.h[1], -, -, -, -
add_member32 v5.8h, v0.h[0], v1.h[2], v3.h[1], v2.h[3], -, -, -, +, 2
add_member32 v6.4h, v2.h[0], v3.h[2], v1.h[1], v0.h[3], -, +, +, +
add_member32 v6.8h, v3.h[2], v0.h[3], v1.h[3], v3.h[1], +, +, +, -, 2
add_member32 v7.4h, v1.h[1], v1.h[3], v2.h[3], v0.h[0], +, +, -, -
add_member32 v7.8h, v0.h[3], v3.h[1], v0.h[1], v3.h[3], +, -, -, +, 2
add_member32 v16.4h, v3.h[0], v0.h[2], v3.h[2], v0.h[1], +, -, -, +
add_member32 v16.8h, v2.h[2], v2.h[0], v1.h[0], v3.h[2], -, -, +, +, 2
add_member32 v17.4h, v0.h[1], v3.h[0], v2.h[0], v0.h[2], -, +, +, -
add_member32 v17.8h, v1.h[3], v0.h[1], v2.h[2], v3.h[0], -, +, -, -, 2
add_member32 v18.4h, v3.h[3], v2.h[1], v0.h[2], v1.h[0], +, +, -, +
add_member32 v18.8h, v1.h[2], v2.h[3], v3.h[3], v2.h[2], +, -, -, +, 2
add_member32 v19.4h, v0.h[2], v0.h[1], v0.h[3], v1.h[2], +, -, +, -
add_member32 v19.8h, v2.h[3], v2.h[2], v2.h[1], v2.h[0], +, -, +, -, 2
ret
endfunc
function tr_block3
multiply v2
add_member32 v4.8h, v1.h[2], v0.h[3], v0.h[0], v0.h[2], -, -, -, -, 2
add_member32 v5.4h, v2.h[2], v3.h[3], v2.h[3], v1.h[2], -, -, +, +
add_member32 v5.8h, v1.h[0], v0.h[2], v2.h[1], v3.h[3], +, +, +, -, 2
add_member32 v6.4h, v3.h[0], v2.h[2], v0.h[1], v1.h[3], +, -, -, -
add_member32 v6.8h, v0.h[2], v2.h[0], v3.h[0], v0.h[0], -, -, +, +, 2
add_member32 v7.4h, v3.h[2], v1.h[0], v2.h[0], v2.h[2], -, +, +, -
add_member32 v7.8h, v0.h[0], v3.h[2], v0.h[2], v3.h[0], +, +, -, -, 2
add_member32 v16.4h, v3.h[3], v0.h[1], v3.h[1], v0.h[3], -, -, +, +
add_member32 v16.8h, v0.h[1], v2.h[3], v1.h[3], v1.h[1], -, +, +, -, 2
add_member32 v17.4h, v3.h[1], v1.h[3], v0.h[3], v3.h[2], +, +, -, +
add_member32 v17.8h, v0.h[3], v1.h[1], v3.h[2], v2.h[0], +, -, +, +, 2
add_member32 v18.4h, v2.h[3], v3.h[1], v1.h[2], v0.h[1], -, -, +, -
add_member32 v18.8h, v1.h[1], v0.h[0], v1.h[0], v2.h[1], -, +, -, +, 2
add_member32 v19.4h, v2.h[1], v3.h[0], v3.h[3], v3.h[1], +, -, +, +
add_member32 v19.8h, v1.h[3], v1.h[2], v1.h[1], v1.h[0], +, -, +, -, 2
ret
endfunc
function tr_block4
multiply v3
add_member32 v4.8h, v1.h[1], v2.h[0], v2.h[3], v3.h[2], -, -, -, -, 2
add_member32 v5.4h, v0.h[0], v0.h[3], v2.h[0], v3.h[1], +, +, +, +
add_member32 v5.8h, v2.h[0], v0.h[0], v1.h[1], v3.h[0], -, -, -, -, 2
add_member32 v6.4h, v3.h[3], v1.h[2], v0.h[2], v2.h[3], +, +, +, +
add_member32 v6.8h, v2.h[1], v2.h[3], v0.h[0], v2.h[2], +, -, -, -, 2
add_member32 v7.4h, v0.h[2], v3.h[3], v0.h[3], v2.h[1], -, -, +, +
add_member32 v7.8h, v1.h[0], v2.h[2], v1.h[2], v2.h[0], +, +, -, -, 2
add_member32 v16.4h, v2.h[3], v1.h[1], v2.h[1], v1.h[3], -, -, +, +
add_member32 v16.8h, v3.h[1], v0.h[1], v3.h[0], v1.h[2], -, +, -, -, 2
add_member32 v17.4h, v1.h[2], v1.h[0], v3.h[3], v1.h[1], +, -, +, +
add_member32 v17.8h, v0.h[1], v2.h[1], v3.h[1], v1.h[0], -, +, +, -, 2
add_member32 v18.4h, v1.h[3], v3.h[2], v2.h[2], v0.h[3], +, -, -, +
add_member32 v18.8h, v3.h[2], v3.h[0], v1.h[3], v0.h[2], -, -, +, -, 2
add_member32 v19.4h, v2.h[2], v1.h[3], v1.h[0], v0.h[1], -, +, -, +
add_member32 v19.8h, v0.h[3], v0.h[2], v0.h[1], v0.h[0], +, -, +, -, 2
ret
endfunc
.macro tr_32x4 name, shift
function func_tr_32x4_\name
mov x10, x30
bl func_tr_16x4_noscale
load32
movrel x9, trans, 32
ld1 {v0.4h-v1.4h}, [x9], #16
ld1 {v2.4h-v3.4h}, [x9]
add x4, sp, #2048
mov x2, #64
mov x8, #-64
bl tr_block1
mov x1, x11
add x3, x11, #(56 + 3 * 64)
scale_store \shift
bl tr_block2
add x1, x11, #8
add x3, x11, #(48 + 3 * 64)
scale_store \shift
bl tr_block3
add x1, x11, #16
add x3, x11, #(40 + 3 * 64)
scale_store \shift
bl tr_block4
add x1, x11, #24
add x3, x11, #(32 + 3 * 64)
scale_store \shift
ret x10
endfunc
.endm
.macro idct_32x32 bitdepth
function ff_hevc_idct_32x32_\bitdepth\()_neon, export=1
mov x15, x30
// allocate a temp buffer
sub sp, sp, #2432
.irp i, 0, 1, 2, 3, 4, 5, 6, 7
add x5, x0, #(8 * \i)
add x11, sp, #(8 * \i * 32)
bl func_tr_32x4_firstpass
.endr
.irp i, 0, 1, 2, 3, 4, 5, 6, 7
add x5, sp, #(8 * \i)
add x11, x0, #(8 * \i * 32)
bl func_tr_32x4_secondpass_\bitdepth
.endr
add sp, sp, #2432
ret x15
endfunc
.endm
idct_4x4 8
idct_4x4 10
idct_8x8 8
idct_8x8 10
tr_16x4 firstpass, 7, 512, 1
tr_16x4 secondpass_8, 20 - 8, 512, 1
tr_16x4 secondpass_10, 20 - 10, 512, 1
tr_16x4 noscale, 0, 2048, 4
idct_16x16 8
idct_16x16 10
.ltorg
tr_32x4 firstpass, 7
tr_32x4 secondpass_8, 20 - 8
tr_32x4 secondpass_10, 20 - 10
.ltorg
idct_32x32 8
idct_32x32 10
.macro tr4_luma_shift r0, r1, r2, r3, shift
saddl v0.4s, \r0, \r2 // c0 = src0 + src2
saddl v1.4s, \r2, \r3 // c1 = src2 + src3
ssubl v2.4s, \r0, \r3 // c2 = src0 - src3
smull v3.4s, \r1, v21.4h // c3 = 74 * src1
saddl v7.4s, \r0, \r3 // src0 + src3
ssubw v7.4s, v7.4s, \r2 // src0 - src2 + src3
mul v7.4s, v7.4s, v18.4s // dst2 = 74 * (src0 - src2 + src3)
mul v5.4s, v0.4s, v19.4s // 29 * c0
mul v6.4s, v1.4s, v20.4s // 55 * c1
add v5.4s, v5.4s, v6.4s // 29 * c0 + 55 * c1
add v5.4s, v5.4s, v3.4s // dst0 = 29 * c0 + 55 * c1 + c3
mul v1.4s, v1.4s, v19.4s // 29 * c1
mul v6.4s, v2.4s, v20.4s // 55 * c2
sub v6.4s, v6.4s, v1.4s // 55 * c2 - 29 * c1
add v6.4s, v6.4s, v3.4s // dst1 = 55 * c2 - 29 * c1 + c3
mul v0.4s, v0.4s, v20.4s // 55 * c0
mul v2.4s, v2.4s, v19.4s // 29 * c2
add v0.4s, v0.4s, v2.4s // 55 * c0 + 29 * c2
sub v0.4s, v0.4s, v3.4s // dst3 = 55 * c0 + 29 * c2 - c3
sqrshrn \r0, v5.4s, \shift
sqrshrn \r1, v6.4s, \shift
sqrshrn \r2, v7.4s, \shift
sqrshrn \r3, v0.4s, \shift
.endm
function ff_hevc_transform_luma_4x4_neon_8, export=1
ld1 {v28.4h-v31.4h}, [x0]
movi v18.4s, #74
movi v19.4s, #29
movi v20.4s, #55
movi v21.4h, #74
tr4_luma_shift v28.4h, v29.4h, v30.4h, v31.4h, #7
transpose_4x4H v28, v29, v30, v31, v22, v23, v24, v25
tr4_luma_shift v28.4h, v29.4h, v30.4h, v31.4h, #12
transpose_4x4H v28, v29, v30, v31, v22, v23, v24, v25
st1 {v28.4h-v31.4h}, [x0]
ret
endfunc
// void ff_hevc_idct_NxN_dc_DEPTH_neon(int16_t *coeffs)
.macro idct_dc size, bitdepth
function ff_hevc_idct_\size\()x\size\()_dc_\bitdepth\()_neon, export=1
ld1r {v4.8h}, [x0]
srshr v4.8h, v4.8h, #1
srshr v0.8h, v4.8h, #(14 - \bitdepth)
srshr v1.8h, v4.8h, #(14 - \bitdepth)
.if \size > 4
srshr v2.8h, v4.8h, #(14 - \bitdepth)
srshr v3.8h, v4.8h, #(14 - \bitdepth)
.if \size > 16 /* dc 32x32 */
mov x2, #4
1:
subs x2, x2, #1
.endif
add x12, x0, #64
mov x13, #128
.if \size > 8 /* dc 16x16 */
st1 {v0.8h-v3.8h}, [x0], x13
st1 {v0.8h-v3.8h}, [x12], x13
st1 {v0.8h-v3.8h}, [x0], x13
st1 {v0.8h-v3.8h}, [x12], x13
st1 {v0.8h-v3.8h}, [x0], x13
st1 {v0.8h-v3.8h}, [x12], x13
.endif /* dc 8x8 */
st1 {v0.8h-v3.8h}, [x0], x13
st1 {v0.8h-v3.8h}, [x12], x13
.if \size > 16 /* dc 32x32 */
bne 1b
.endif
.else /* dc 4x4 */
st1 {v0.8h-v1.8h}, [x0]
.endif
ret
endfunc
.endm
idct_dc 4, 8
idct_dc 4, 10
idct_dc 8, 8
idct_dc 8, 10
idct_dc 16, 8
idct_dc 16, 10
idct_dc 32, 8
idct_dc 32, 10
@@ -0,0 +1,271 @@
/*
* Copyright (c) 2020 Reimar Döffinger
* Copyright (c) 2023 xu fulong <839789740@qq.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/aarch64/h26x/dsp.h"
#include "libavcodec/hevc/dsp.h"
void ff_hevc_v_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_v_loop_filter_chroma_10_neon(uint8_t *_pix, ptrdiff_t _stride,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_v_loop_filter_chroma_12_neon(uint8_t *_pix, ptrdiff_t _stride,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_h_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_h_loop_filter_chroma_10_neon(uint8_t *_pix, ptrdiff_t _stride,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_h_loop_filter_chroma_12_neon(uint8_t *_pix, ptrdiff_t _stride,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_v_loop_filter_luma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_v_loop_filter_luma_10_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_v_loop_filter_luma_12_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_h_loop_filter_luma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_h_loop_filter_luma_10_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_h_loop_filter_luma_12_neon(uint8_t *_pix, ptrdiff_t _stride, int beta,
const int *_tc, const uint8_t *_no_p, const uint8_t *_no_q);
void ff_hevc_add_residual_4x4_8_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_4x4_10_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_4x4_12_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_8x8_8_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_8x8_10_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_8x8_12_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_16x16_8_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_16x16_10_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_16x16_12_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_32x32_8_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_32x32_10_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_add_residual_32x32_12_neon(uint8_t *_dst, const int16_t *coeffs,
ptrdiff_t stride);
void ff_hevc_idct_4x4_8_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_4x4_10_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_8x8_8_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_8x8_10_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_16x16_8_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_16x16_10_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_32x32_8_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_32x32_10_neon(int16_t *coeffs, int col_limit);
void ff_hevc_idct_4x4_dc_8_neon(int16_t *coeffs);
void ff_hevc_idct_8x8_dc_8_neon(int16_t *coeffs);
void ff_hevc_idct_16x16_dc_8_neon(int16_t *coeffs);
void ff_hevc_idct_32x32_dc_8_neon(int16_t *coeffs);
void ff_hevc_idct_4x4_dc_10_neon(int16_t *coeffs);
void ff_hevc_idct_8x8_dc_10_neon(int16_t *coeffs);
void ff_hevc_idct_16x16_dc_10_neon(int16_t *coeffs);
void ff_hevc_idct_32x32_dc_10_neon(int16_t *coeffs);
void ff_hevc_transform_luma_4x4_neon_8(int16_t *coeffs);
#define NEON8_FNASSIGN(member, v, h, fn, ext) \
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \
member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
member[8][v][h] = ff_hevc_put_hevc_##fn##48_8_neon##ext; \
member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
#define NEON8_FNASSIGN_SHARED_32(member, v, h, fn, ext) \
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \
member[7][v][h] = \
member[8][v][h] = \
member[9][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext;
#define NEON8_FNASSIGN_PARTIAL_4(member, v, h, fn, ext) \
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
member[7][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
member[8][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
#define NEON8_FNASSIGN_PARTIAL_5(member, v, h, fn, ext) \
member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (!have_neon(cpu_flags)) return;
if (bit_depth == 8) {
c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_neon;
c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_neon;
c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_neon;
c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_neon;
c->add_residual[0] = ff_hevc_add_residual_4x4_8_neon;
c->add_residual[1] = ff_hevc_add_residual_8x8_8_neon;
c->add_residual[2] = ff_hevc_add_residual_16x16_8_neon;
c->add_residual[3] = ff_hevc_add_residual_32x32_8_neon;
c->idct[0] = ff_hevc_idct_4x4_8_neon;
c->idct[1] = ff_hevc_idct_8x8_8_neon;
c->idct[2] = ff_hevc_idct_16x16_8_neon;
c->idct[3] = ff_hevc_idct_32x32_8_neon;
c->idct_dc[0] = ff_hevc_idct_4x4_dc_8_neon;
c->idct_dc[1] = ff_hevc_idct_8x8_dc_8_neon;
c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon;
c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_neon;
c->transform_4x4_luma = ff_hevc_transform_luma_4x4_neon_8;
c->sao_band_filter[0] =
c->sao_band_filter[1] =
c->sao_band_filter[2] =
c->sao_band_filter[3] =
c->sao_band_filter[4] = ff_h26x_sao_band_filter_8x8_8_neon;
c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8x8_8_neon;
c->sao_edge_filter[1] =
c->sao_edge_filter[2] =
c->sao_edge_filter[3] =
c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_16x16_8_neon;
c->put_hevc_qpel[1][0][1] = ff_hevc_put_hevc_qpel_h4_8_neon;
c->put_hevc_qpel[2][0][1] = ff_hevc_put_hevc_qpel_h6_8_neon;
c->put_hevc_qpel[3][0][1] = ff_hevc_put_hevc_qpel_h8_8_neon;
c->put_hevc_qpel[4][0][1] =
c->put_hevc_qpel[6][0][1] = ff_hevc_put_hevc_qpel_h12_8_neon;
c->put_hevc_qpel[5][0][1] = ff_hevc_put_hevc_qpel_h16_8_neon;
c->put_hevc_qpel[7][0][1] =
c->put_hevc_qpel[8][0][1] =
c->put_hevc_qpel[9][0][1] = ff_hevc_put_hevc_qpel_h32_8_neon;
c->put_hevc_qpel_uni[1][0][1] = ff_hevc_put_hevc_qpel_uni_h4_8_neon;
c->put_hevc_qpel_uni[2][0][1] = ff_hevc_put_hevc_qpel_uni_h6_8_neon;
c->put_hevc_qpel_uni[3][0][1] = ff_hevc_put_hevc_qpel_uni_h8_8_neon;
c->put_hevc_qpel_uni[4][0][1] =
c->put_hevc_qpel_uni[6][0][1] = ff_hevc_put_hevc_qpel_uni_h12_8_neon;
c->put_hevc_qpel_uni[5][0][1] = ff_hevc_put_hevc_qpel_uni_h16_8_neon;
c->put_hevc_qpel_uni[7][0][1] =
c->put_hevc_qpel_uni[8][0][1] =
c->put_hevc_qpel_uni[9][0][1] = ff_hevc_put_hevc_qpel_uni_h32_8_neon;
c->put_hevc_qpel_bi[1][0][1] = ff_hevc_put_hevc_qpel_bi_h4_8_neon;
c->put_hevc_qpel_bi[2][0][1] = ff_hevc_put_hevc_qpel_bi_h6_8_neon;
c->put_hevc_qpel_bi[3][0][1] = ff_hevc_put_hevc_qpel_bi_h8_8_neon;
c->put_hevc_qpel_bi[4][0][1] =
c->put_hevc_qpel_bi[6][0][1] = ff_hevc_put_hevc_qpel_bi_h12_8_neon;
c->put_hevc_qpel_bi[5][0][1] = ff_hevc_put_hevc_qpel_bi_h16_8_neon;
c->put_hevc_qpel_bi[7][0][1] =
c->put_hevc_qpel_bi[8][0][1] =
c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_hevc_qpel_bi_h32_8_neon;
NEON8_FNASSIGN(c->put_hevc_epel, 0, 0, pel_pixels,);
NEON8_FNASSIGN(c->put_hevc_epel, 1, 0, epel_v,);
NEON8_FNASSIGN(c->put_hevc_qpel, 0, 0, pel_pixels,);
NEON8_FNASSIGN(c->put_hevc_qpel, 1, 0, qpel_v,);
NEON8_FNASSIGN(c->put_hevc_epel_bi, 0, 0, pel_bi_pixels,);
NEON8_FNASSIGN(c->put_hevc_epel_bi, 0, 1, epel_bi_h,);
NEON8_FNASSIGN(c->put_hevc_epel_bi, 1, 0, epel_bi_v,);
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 0, 0, pel_bi_pixels,);
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 0, qpel_bi_v,);
NEON8_FNASSIGN(c->put_hevc_epel_uni, 0, 0, pel_uni_pixels,);
NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 0, epel_uni_v,);
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 0, 0, pel_uni_pixels,);
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 0, qpel_uni_v,);
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 0, pel_uni_w_pixels,);
NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 0, pel_uni_w_pixels,);
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 0, epel_uni_w_v,);
NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);
NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,);
NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h,);
NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv,);
NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 1, epel_uni_hv,);
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 1, epel_uni_w_hv,);
NEON8_FNASSIGN(c->put_hevc_epel_bi, 1, 1, epel_bi_hv,);
NEON8_FNASSIGN_SHARED_32(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h,);
NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv,);
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 1, qpel_uni_hv,);
NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1, qpel_uni_w_hv,);
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 1, qpel_bi_hv,);
if (have_i8mm(cpu_flags)) {
NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);
NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 1, epel_uni_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm);
NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 1, epel_uni_w_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_epel_bi, 1, 1, epel_bi_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm);
NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 1, qpel_uni_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm);
NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1, qpel_uni_w_hv, _i8mm);
NEON8_FNASSIGN(c->put_hevc_qpel_bi, 1, 1, qpel_bi_hv, _i8mm);
}
}
if (bit_depth == 10) {
c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_neon;
c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_neon;
c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_neon;
c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_neon;
c->add_residual[0] = ff_hevc_add_residual_4x4_10_neon;
c->add_residual[1] = ff_hevc_add_residual_8x8_10_neon;
c->add_residual[2] = ff_hevc_add_residual_16x16_10_neon;
c->add_residual[3] = ff_hevc_add_residual_32x32_10_neon;
c->idct[0] = ff_hevc_idct_4x4_10_neon;
c->idct[1] = ff_hevc_idct_8x8_10_neon;
c->idct[2] = ff_hevc_idct_16x16_10_neon;
c->idct[3] = ff_hevc_idct_32x32_10_neon;
c->idct_dc[0] = ff_hevc_idct_4x4_dc_10_neon;
c->idct_dc[1] = ff_hevc_idct_8x8_dc_10_neon;
c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_neon;
c->idct_dc[3] = ff_hevc_idct_32x32_dc_10_neon;
}
if (bit_depth == 12) {
c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_neon;
c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_neon;
c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_neon;
c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_neon;
c->add_residual[0] = ff_hevc_add_residual_4x4_12_neon;
c->add_residual[1] = ff_hevc_add_residual_8x8_12_neon;
c->add_residual[2] = ff_hevc_add_residual_16x16_12_neon;
c->add_residual[3] = ff_hevc_add_residual_32x32_12_neon;
}
}
@@ -0,0 +1,123 @@
/*
* ARM NEON optimised DSP functions
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stddef.h>
#include <stdint.h>
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/hpeldsp.h"
void ff_put_pixels16_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_x2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_y2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_xy2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_x2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_y2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_xy2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_x2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_y2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels16_xy2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_x2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_y2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_put_pixels8_xy2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_x2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_y2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_xy2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels8_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels8_x2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels8_y2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels8_xy2_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_x2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_y2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
void ff_avg_pixels16_xy2_no_rnd_neon(uint8_t *block, const uint8_t *pixels,
ptrdiff_t line_size, int h);
av_cold void ff_hpeldsp_init_aarch64(HpelDSPContext *c, int flags)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
c->put_pixels_tab[0][0] = ff_put_pixels16_neon;
c->put_pixels_tab[0][1] = ff_put_pixels16_x2_neon;
c->put_pixels_tab[0][2] = ff_put_pixels16_y2_neon;
c->put_pixels_tab[0][3] = ff_put_pixels16_xy2_neon;
c->put_pixels_tab[1][0] = ff_put_pixels8_neon;
c->put_pixels_tab[1][1] = ff_put_pixels8_x2_neon;
c->put_pixels_tab[1][2] = ff_put_pixels8_y2_neon;
c->put_pixels_tab[1][3] = ff_put_pixels8_xy2_neon;
c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_neon;
c->put_no_rnd_pixels_tab[0][1] = ff_put_pixels16_x2_no_rnd_neon;
c->put_no_rnd_pixels_tab[0][2] = ff_put_pixels16_y2_no_rnd_neon;
c->put_no_rnd_pixels_tab[0][3] = ff_put_pixels16_xy2_no_rnd_neon;
c->put_no_rnd_pixels_tab[1][0] = ff_put_pixels8_neon;
c->put_no_rnd_pixels_tab[1][1] = ff_put_pixels8_x2_no_rnd_neon;
c->put_no_rnd_pixels_tab[1][2] = ff_put_pixels8_y2_no_rnd_neon;
c->put_no_rnd_pixels_tab[1][3] = ff_put_pixels8_xy2_no_rnd_neon;
c->avg_pixels_tab[0][0] = ff_avg_pixels16_neon;
c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_neon;
c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_neon;
c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_neon;
c->avg_pixels_tab[1][0] = ff_avg_pixels8_neon;
c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_neon;
c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_neon;
c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_neon;
c->avg_no_rnd_pixels_tab[0] = ff_avg_pixels16_neon;
c->avg_no_rnd_pixels_tab[1] = ff_avg_pixels16_x2_no_rnd_neon;
c->avg_no_rnd_pixels_tab[2] = ff_avg_pixels16_y2_no_rnd_neon;
c->avg_no_rnd_pixels_tab[3] = ff_avg_pixels16_xy2_no_rnd_neon;
}
}
+399
View File
@@ -0,0 +1,399 @@
/*
* ARM NEON optimised DSP functions
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2013 Janne Grunau <janne-libav@jannau.net>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
.macro pixels16 rnd=1, avg=0
.if \avg
mov x12, x0
.endif
1: ld1 {v0.16b}, [x1], x2
ld1 {v1.16b}, [x1], x2
ld1 {v2.16b}, [x1], x2
ld1 {v3.16b}, [x1], x2
.if \avg
ld1 {v4.16b}, [x12], x2
urhadd v0.16b, v0.16b, v4.16b
ld1 {v5.16b}, [x12], x2
urhadd v1.16b, v1.16b, v5.16b
ld1 {v6.16b}, [x12], x2
urhadd v2.16b, v2.16b, v6.16b
ld1 {v7.16b}, [x12], x2
urhadd v3.16b, v3.16b, v7.16b
.endif
subs w3, w3, #4
st1 {v0.16b}, [x0], x2
st1 {v1.16b}, [x0], x2
st1 {v2.16b}, [x0], x2
st1 {v3.16b}, [x0], x2
b.ne 1b
ret
.endm
.macro pixels16_x2 rnd=1, avg=0
1:
ldur q1, [x1, #1]
ld1 {v0.16b}, [x1], x2
subs w3, w3, #2
ldur q3, [x1, #1]
ld1 {v2.16b}, [x1], x2
avg v0.16b, v0.16b, v1.16b
avg v2.16b, v2.16b, v3.16b
.if \avg
ld1 {v1.16b}, [x0], x2
ld1 {v3.16b}, [x0]
urhadd v0.16b, v0.16b, v1.16b
urhadd v2.16b, v2.16b, v3.16b
sub x0, x0, x2
.endif
st1 {v0.16b}, [x0], x2
st1 {v2.16b}, [x0], x2
b.ne 1b
ret
.endm
.macro pixels16_y2 rnd=1, avg=0
sub w3, w3, #2
ld1 {v0.16b}, [x1], x2
ld1 {v1.16b}, [x1], x2
1: subs w3, w3, #2
avg v2.16b, v0.16b, v1.16b
ld1 {v0.16b}, [x1], x2
avg v3.16b, v0.16b, v1.16b
ld1 {v1.16b}, [x1], x2
.if \avg
ld1 {v4.16b}, [x0], x2
ld1 {v5.16b}, [x0]
urhadd v2.16b, v2.16b, v4.16b
urhadd v3.16b, v3.16b, v5.16b
sub x0, x0, x2
.endif
st1 {v2.16b}, [x0], x2
st1 {v3.16b}, [x0], x2
b.ne 1b
avg v2.16b, v0.16b, v1.16b
ld1 {v0.16b}, [x1], x2
avg v3.16b, v0.16b, v1.16b
.if \avg
ld1 {v4.16b}, [x0], x2
ld1 {v5.16b}, [x0]
urhadd v2.16b, v2.16b, v4.16b
urhadd v3.16b, v3.16b, v5.16b
sub x0, x0, x2
.endif
st1 {v2.16b}, [x0], x2
st1 {v3.16b}, [x0], x2
ret
.endm
.macro pixels16_xy2 rnd=1, avg=0
sub w3, w3, #2
ldur q1, [x1, #1]
ld1 {v0.16b}, [x1], x2
NRND movi v26.8H, #1
ldur q5, [x1, #1]
ld1 {v4.16b}, [x1], x2
uaddl v16.8h, v0.8b, v1.8b
uaddl2 v20.8h, v0.16b, v1.16b
uaddl v18.8h, v4.8b, v5.8b
uaddl2 v22.8h, v4.16b, v5.16b
1: subs w3, w3, #2
ldur q30, [x1, #1]
ld1 {v0.16b}, [x1], x2
add v24.8h, v16.8h, v18.8h
NRND add v24.8H, v24.8H, v26.8H
add v1.8h, v20.8h, v22.8h
mshrn v28.8b, v24.8h, #2
NRND add v1.8H, v1.8H, v26.8H
mshrn2 v28.16b, v1.8h, #2
.if \avg
ld1 {v16.16b}, [x0]
urhadd v28.16b, v28.16b, v16.16b
.endif
uaddl v16.8h, v0.8b, v30.8b
ldur q3, [x1, #1]
ld1 {v2.16b}, [x1], x2
uaddl2 v20.8h, v0.16b, v30.16b
st1 {v28.16b}, [x0], x2
add v24.8h, v16.8h, v18.8h
NRND add v24.8H, v24.8H, v26.8H
add v0.8h, v20.8h, v22.8h
mshrn v30.8b, v24.8h, #2
NRND add v0.8H, v0.8H, v26.8H
mshrn2 v30.16b, v0.8h, #2
.if \avg
ld1 {v18.16b}, [x0]
urhadd v30.16b, v30.16b, v18.16b
.endif
uaddl v18.8h, v2.8b, v3.8b
uaddl2 v22.8h, v2.16b, v3.16b
st1 {v30.16b}, [x0], x2
b.gt 1b
ldur q30, [x1, #1]
ld1 {v0.16b}, [x1], x2
add v24.8h, v16.8h, v18.8h
NRND add v24.8H, v24.8H, v26.8H
add v1.8h, v20.8h, v22.8h
mshrn v28.8b, v24.8h, #2
NRND add v1.8H, v1.8H, v26.8H
mshrn2 v28.16b, v1.8h, #2
.if \avg
ld1 {v16.16b}, [x0]
urhadd v28.16b, v28.16b, v16.16b
.endif
uaddl v16.8h, v0.8b, v30.8b
uaddl2 v20.8h, v0.16b, v30.16b
st1 {v28.16b}, [x0], x2
add v24.8h, v16.8h, v18.8h
NRND add v24.8H, v24.8H, v26.8H
add v0.8h, v20.8h, v22.8h
mshrn v30.8b, v24.8h, #2
NRND add v0.8H, v0.8H, v26.8H
mshrn2 v30.16b, v0.8h, #2
.if \avg
ld1 {v18.16b}, [x0]
urhadd v30.16b, v30.16b, v18.16b
.endif
st1 {v30.16b}, [x0], x2
ret
.endm
.macro pixels8 rnd=1, avg=0
1: ld1 {v0.8b}, [x1], x2
ld1 {v1.8b}, [x1], x2
ld1 {v2.8b}, [x1], x2
ld1 {v3.8b}, [x1], x2
.if \avg
ld1 {v4.8b}, [x0], x2
urhadd v0.8b, v0.8b, v4.8b
ld1 {v5.8b}, [x0], x2
urhadd v1.8b, v1.8b, v5.8b
ld1 {v6.8b}, [x0], x2
urhadd v2.8b, v2.8b, v6.8b
ld1 {v7.8b}, [x0], x2
urhadd v3.8b, v3.8b, v7.8b
sub x0, x0, x2, lsl #2
.endif
subs w3, w3, #4
st1 {v0.8b}, [x0], x2
st1 {v1.8b}, [x0], x2
st1 {v2.8b}, [x0], x2
st1 {v3.8b}, [x0], x2
b.ne 1b
ret
.endm
.macro pixels8_x2 rnd=1, avg=0
1:
ldur d1, [x1, #1]
ld1 {v0.8b}, [x1], x2
ldur d3, [x1, #1]
ld1 {v2.8b}, [x1], x2
subs w3, w3, #2
avg v0.8b, v0.8b, v1.8b
avg v2.8b, v2.8b, v3.8b
.if \avg
ld1 {v4.8b}, [x0], x2
ld1 {v5.8b}, [x0]
urhadd v0.8b, v0.8b, v4.8b
urhadd v2.8b, v2.8b, v5.8b
sub x0, x0, x2
.endif
st1 {v0.8b}, [x0], x2
st1 {v2.8b}, [x0], x2
b.ne 1b
ret
.endm
.macro pixels8_y2 rnd=1, avg=0
sub w3, w3, #2
ld1 {v0.8b}, [x1], x2
ld1 {v1.8b}, [x1], x2
1: subs w3, w3, #2
avg v4.8b, v0.8b, v1.8b
ld1 {v0.8b}, [x1], x2
avg v5.8b, v0.8b, v1.8b
ld1 {v1.8b}, [x1], x2
.if \avg
ld1 {v2.8b}, [x0], x2
ld1 {v3.8b}, [x0]
urhadd v4.8b, v4.8b, v2.8b
urhadd v5.8b, v5.8b, v3.8b
sub x0, x0, x2
.endif
st1 {v4.8b}, [x0], x2
st1 {v5.8b}, [x0], x2
b.ne 1b
avg v4.8b, v0.8b, v1.8b
ld1 {v0.8b}, [x1], x2
avg v5.8b, v0.8b, v1.8b
.if \avg
ld1 {v2.8b}, [x0], x2
ld1 {v3.8b}, [x0]
urhadd v4.8b, v4.8b, v2.8b
urhadd v5.8b, v5.8b, v3.8b
sub x0, x0, x2
.endif
st1 {v4.8b}, [x0], x2
st1 {v5.8b}, [x0], x2
ret
.endm
.macro pixels8_xy2 rnd=1, avg=0
ldur d4, [x1, #1]
sub w3, w3, #2
ld1 {v0.8b}, [x1], x2
NRND movi v19.8H, #1
ldur d6, [x1, #1]
ld1 {v1.8b}, [x1], x2
uaddl v16.8h, v0.8b, v4.8b
uaddl v17.8h, v1.8b, v6.8b
1: subs w3, w3, #2
ldur d4, [x1, #1]
ld1 {v0.8b}, [x1], x2
add v18.8h, v16.8h, v17.8h
NRND add v18.8H, v18.8H, v19.8H
uaddl v16.8h, v0.8b, v4.8b
mshrn v5.8b, v18.8h, #2
ldur d6, [x1, #1]
ld1 {v1.8b}, [x1], x2
add v18.8h, v16.8h, v17.8h
.if \avg
ld1 {v7.8b}, [x0]
urhadd v5.8b, v5.8b, v7.8b
.endif
NRND add v18.8H, v18.8H, v19.8H
st1 {v5.8b}, [x0], x2
mshrn v7.8b, v18.8h, #2
.if \avg
ld1 {v5.8b}, [x0]
urhadd v7.8b, v7.8b, v5.8b
.endif
uaddl v17.8h, v1.8b, v6.8b
st1 {v7.8b}, [x0], x2
b.gt 1b
ldur d4, [x1, #1]
ld1 {v0.8b}, [x1], x2
add v18.8h, v16.8h, v17.8h
NRND add v18.8H, v18.8H, v19.8H
uaddl v16.8h, v0.8b, v4.8b
mshrn v5.8b, v18.8h, #2
add v18.8h, v16.8h, v17.8h
.if \avg
ld1 {v7.8b}, [x0]
urhadd v5.8b, v5.8b, v7.8b
.endif
NRND add v18.8H, v18.8H, v19.8H
st1 {v5.8b}, [x0], x2
mshrn v7.8b, v18.8h, #2
.if \avg
ld1 {v5.8b}, [x0]
urhadd v7.8b, v7.8b, v5.8b
.endif
st1 {v7.8b}, [x0], x2
ret
.endm
.macro pixfunc pfx, name, suf, rnd=1, avg=0
.if \rnd
.macro avg rd, rn, rm
urhadd \rd, \rn, \rm
.endm
.macro mshrn rd, rn, rm
rshrn \rd, \rn, \rm
.endm
.macro mshrn2 rd, rn, rm
rshrn2 \rd, \rn, \rm
.endm
.macro NRND insn:vararg
.endm
.else
.macro avg rd, rn, rm
uhadd \rd, \rn, \rm
.endm
.macro mshrn rd, rn, rm
shrn \rd, \rn, \rm
.endm
.macro mshrn2 rd, rn, rm
shrn2 \rd, \rn, \rm
.endm
.macro NRND insn:vararg
\insn
.endm
.endif
function ff_\pfx\name\suf\()_neon, export=1
\name \rnd, \avg
endfunc
.purgem avg
.purgem mshrn
.purgem mshrn2
.purgem NRND
.endm
.macro pixfunc2 pfx, name, avg=0
pixfunc \pfx, \name, rnd=1, avg=\avg
pixfunc \pfx, \name, _no_rnd, rnd=0, avg=\avg
.endm
function ff_put_h264_qpel16_mc00_neon, export=1
mov w3, #16
endfunc
pixfunc put_, pixels16, avg=0
pixfunc2 put_, pixels16_x2, avg=0
pixfunc2 put_, pixels16_y2, avg=0
pixfunc2 put_, pixels16_xy2, avg=0
function ff_avg_h264_qpel16_mc00_neon, export=1
mov w3, #16
endfunc
pixfunc avg_, pixels16, avg=1
pixfunc2 avg_, pixels16_x2, avg=1
pixfunc2 avg_, pixels16_y2, avg=1
pixfunc2 avg_, pixels16_xy2, avg=1
function ff_put_h264_qpel8_mc00_neon, export=1
mov w3, #8
endfunc
pixfunc put_, pixels8, avg=0
pixfunc2 put_, pixels8_x2, avg=0
pixfunc2 put_, pixels8_y2, avg=0
pixfunc2 put_, pixels8_xy2, avg=0
function ff_avg_h264_qpel8_mc00_neon, export=1
mov w3, #8
endfunc
pixfunc avg_, pixels8, avg=1
pixfunc avg_, pixels8_x2, avg=1
pixfunc avg_, pixels8_y2, avg=1
pixfunc avg_, pixels8_xy2, avg=1
+29
View File
@@ -0,0 +1,29 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_AARCH64_IDCT_H
#define AVCODEC_AARCH64_IDCT_H
#include <stddef.h>
#include <stdint.h>
void ff_simple_idct_neon(int16_t *data);
void ff_simple_idct_put_neon(uint8_t *dest, ptrdiff_t line_size, int16_t *data);
void ff_simple_idct_add_neon(uint8_t *dest, ptrdiff_t line_size, int16_t *data);
#endif /* AVCODEC_AARCH64_IDCT_H */
@@ -0,0 +1,55 @@
/*
* ARM-NEON-optimized IDCT functions
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2017 Matthieu Bouron <matthieu.bouron@gmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/idctdsp.h"
#include "idct.h"
void ff_put_pixels_clamped_neon(const int16_t *, uint8_t *, ptrdiff_t);
void ff_put_signed_pixels_clamped_neon(const int16_t *, uint8_t *, ptrdiff_t);
void ff_add_pixels_clamped_neon(const int16_t *, uint8_t *, ptrdiff_t);
av_cold void ff_idctdsp_init_aarch64(IDCTDSPContext *c, AVCodecContext *avctx,
unsigned high_bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
if (!avctx->lowres && !high_bit_depth) {
if (avctx->idct_algo == FF_IDCT_AUTO ||
avctx->idct_algo == FF_IDCT_SIMPLEAUTO ||
avctx->idct_algo == FF_IDCT_SIMPLENEON) {
c->idct_put = ff_simple_idct_put_neon;
c->idct_add = ff_simple_idct_add_neon;
c->idct = ff_simple_idct_neon;
c->perm_type = FF_IDCT_PERM_PARTTRANS;
}
}
c->add_pixels_clamped = ff_add_pixels_clamped_neon;
c->put_pixels_clamped = ff_put_pixels_clamped_neon;
c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_neon;
}
}
+130
View File
@@ -0,0 +1,130 @@
/*
* IDCT AArch64 NEON optimisations
*
* Copyright (c) 2022 Ben Avison <bavison@riscosopen.org>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
// Clamp 16-bit signed block coefficients to unsigned 8-bit
// On entry:
// x0 -> array of 64x 16-bit coefficients
// x1 -> 8-bit results
// x2 = row stride for results, bytes
function ff_put_pixels_clamped_neon, export=1
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0]
sqxtun v0.8b, v0.8h
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
sqxtun v3.8b, v3.8h
sqxtun v4.8b, v4.8h
st1 {v0.8b}, [x1], x2
sqxtun v0.8b, v5.8h
st1 {v1.8b}, [x1], x2
sqxtun v1.8b, v6.8h
st1 {v2.8b}, [x1], x2
sqxtun v2.8b, v7.8h
st1 {v3.8b}, [x1], x2
st1 {v4.8b}, [x1], x2
st1 {v0.8b}, [x1], x2
st1 {v1.8b}, [x1], x2
st1 {v2.8b}, [x1]
ret
endfunc
// Clamp 16-bit signed block coefficients to signed 8-bit (biased by 128)
// On entry:
// x0 -> array of 64x 16-bit coefficients
// x1 -> 8-bit results
// x2 = row stride for results, bytes
function ff_put_signed_pixels_clamped_neon, export=1
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
movi v4.8b, #128
ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x0]
sqxtn v0.8b, v0.8h
sqxtn v1.8b, v1.8h
sqxtn v2.8b, v2.8h
sqxtn v3.8b, v3.8h
sqxtn v5.8b, v16.8h
add v0.8b, v0.8b, v4.8b
sqxtn v6.8b, v17.8h
add v1.8b, v1.8b, v4.8b
sqxtn v7.8b, v18.8h
add v2.8b, v2.8b, v4.8b
sqxtn v16.8b, v19.8h
add v3.8b, v3.8b, v4.8b
st1 {v0.8b}, [x1], x2
add v0.8b, v5.8b, v4.8b
st1 {v1.8b}, [x1], x2
add v1.8b, v6.8b, v4.8b
st1 {v2.8b}, [x1], x2
add v2.8b, v7.8b, v4.8b
st1 {v3.8b}, [x1], x2
add v3.8b, v16.8b, v4.8b
st1 {v0.8b}, [x1], x2
st1 {v1.8b}, [x1], x2
st1 {v2.8b}, [x1], x2
st1 {v3.8b}, [x1]
ret
endfunc
// Add 16-bit signed block coefficients to unsigned 8-bit
// On entry:
// x0 -> array of 64x 16-bit coefficients
// x1 -> 8-bit input and results
// x2 = row stride for 8-bit input and results, bytes
function ff_add_pixels_clamped_neon, export=1
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
mov x3, x1
ld1 {v4.8b}, [x1], x2
ld1 {v5.8b}, [x1], x2
ld1 {v6.8b}, [x1], x2
ld1 {v7.8b}, [x1], x2
ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x0]
uaddw v0.8h, v0.8h, v4.8b
uaddw v1.8h, v1.8h, v5.8b
uaddw v2.8h, v2.8h, v6.8b
ld1 {v4.8b}, [x1], x2
uaddw v3.8h, v3.8h, v7.8b
ld1 {v5.8b}, [x1], x2
sqxtun v0.8b, v0.8h
ld1 {v6.8b}, [x1], x2
sqxtun v1.8b, v1.8h
ld1 {v7.8b}, [x1]
sqxtun v2.8b, v2.8h
sqxtun v3.8b, v3.8h
uaddw v4.8h, v16.8h, v4.8b
st1 {v0.8b}, [x3], x2
uaddw v0.8h, v17.8h, v5.8b
st1 {v1.8b}, [x3], x2
uaddw v1.8h, v18.8h, v6.8b
st1 {v2.8b}, [x3], x2
uaddw v2.8h, v19.8h, v7.8b
sqxtun v4.8b, v4.8h
sqxtun v0.8b, v0.8h
st1 {v3.8b}, [x3], x2
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
st1 {v4.8b}, [x3], x2
st1 {v0.8b}, [x3], x2
st1 {v1.8b}, [x3], x2
st1 {v2.8b}, [x3]
ret
endfunc
@@ -0,0 +1,148 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/mpegvideo.h"
int ff_pix_abs16_neon(MpegEncContext *s, const uint8_t *blk1, const uint8_t *blk2,
ptrdiff_t stride, int h);
int ff_pix_abs16_xy2_neon(MpegEncContext *s, const uint8_t *blk1, const uint8_t *blk2,
ptrdiff_t stride, int h);
int ff_pix_abs16_x2_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_pix_abs16_y2_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_pix_abs8_neon(MpegEncContext *s, const uint8_t *blk1, const uint8_t *blk2,
ptrdiff_t stride, int h);
int sse16_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int sse8_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int sse4_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int vsad16_neon(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h);
int vsad_intra16_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy,
ptrdiff_t stride, int h) ;
int vsad_intra8_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy,
ptrdiff_t stride, int h) ;
int vsse16_neon(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h);
int vsse_intra16_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy,
ptrdiff_t stride, int h);
int nsse16_neon(int multiplier, const uint8_t *s, const uint8_t *s2,
ptrdiff_t stride, int h);
int nsse16_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h);
int pix_median_abs16_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int pix_median_abs8_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_pix_abs8_x2_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_pix_abs8_y2_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int ff_pix_abs8_xy2_neon(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int nsse8_neon(int multiplier, const uint8_t *s, const uint8_t *s2,
ptrdiff_t stride, int h);
int nsse8_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h);
int vsse8_neon(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h);
int vsse_intra8_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy,
ptrdiff_t stride, int h);
#if HAVE_DOTPROD
int sse16_neon_dotprod(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2,
ptrdiff_t stride, int h);
int vsse_intra16_neon_dotprod(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h);
#endif
av_cold void ff_me_cmp_init_aarch64(MECmpContext *c, AVCodecContext *avctx)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
c->pix_abs[0][0] = ff_pix_abs16_neon;
c->pix_abs[0][1] = ff_pix_abs16_x2_neon;
c->pix_abs[0][2] = ff_pix_abs16_y2_neon;
c->pix_abs[0][3] = ff_pix_abs16_xy2_neon;
c->pix_abs[1][0] = ff_pix_abs8_neon;
c->pix_abs[1][1] = ff_pix_abs8_x2_neon;
c->pix_abs[1][2] = ff_pix_abs8_y2_neon;
c->pix_abs[1][3] = ff_pix_abs8_xy2_neon;
c->sad[0] = ff_pix_abs16_neon;
c->sad[1] = ff_pix_abs8_neon;
c->sse[0] = sse16_neon;
c->sse[1] = sse8_neon;
c->sse[2] = sse4_neon;
c->vsad[0] = vsad16_neon;
c->vsad[4] = vsad_intra16_neon;
c->vsad[5] = vsad_intra8_neon;
c->vsse[0] = vsse16_neon;
c->vsse[1] = vsse8_neon;
c->vsse[4] = vsse_intra16_neon;
c->vsse[5] = vsse_intra8_neon;
c->nsse[0] = nsse16_neon_wrapper;
c->nsse[1] = nsse8_neon_wrapper;
c->median_sad[0] = pix_median_abs16_neon;
c->median_sad[1] = pix_median_abs8_neon;
}
#if HAVE_DOTPROD
if (have_dotprod(cpu_flags)) {
c->sse[0] = sse16_neon_dotprod;
c->vsse[4] = vsse_intra16_neon_dotprod;
}
#endif
}
int nsse16_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h)
{
if (c)
return nsse16_neon(c->avctx->nsse_weight, s1, s2, stride, h);
else
return nsse16_neon(8, s1, s2, stride, h);
}
int nsse8_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2,
ptrdiff_t stride, int h)
{
if (c)
return nsse8_neon(c->avctx->nsse_weight, s1, s2, stride, h);
else
return nsse8_neon(8, s1, s2, stride, h);
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,40 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stddef.h>
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/mpegaudiodsp.h"
#include "config.h"
void ff_mpadsp_apply_window_fixed_neon(int32_t *synth_buf, int32_t *window,
int *dither, int16_t *samples, ptrdiff_t incr);
void ff_mpadsp_apply_window_float_neon(float *synth_buf, float *window,
int *dither, float *samples, ptrdiff_t incr);
av_cold void ff_mpadsp_init_aarch64(MPADSPContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
s->apply_window_fixed = ff_mpadsp_apply_window_fixed_neon;
s->apply_window_float = ff_mpadsp_apply_window_float_neon;
}
}
@@ -0,0 +1,225 @@
/*
* Copyright (c) 2014 Janne Grunau <janne-libav@jannau.net>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#define FRAC_BITS 23 // fractional bits for sb_samples and dct
#define WFRAC_BITS 16 // fractional bits for window
#define OUT_SHIFT (WFRAC_BITS + FRAC_BITS - 15)
const tbl_rev128_s, align=4
.byte 12, 13, 14, 15
.byte 8, 9, 10, 11
.byte 4, 5, 6, 7
.byte 0, 1, 2, 3
endconst
.macro apply_window type, st
function ff_mpadsp_apply_window_\type\()_neon, export=1
mov x7, x0
add x8, x0, #512<<2
ld1 {v0.4s,v1.4s,v2.4s,v3.4s}, [x7], #64
ld1 {v4.4s,v5.4s,v6.4s,v7.4s}, [x7], #64
st1 {v0.4s,v1.4s,v2.4s,v3.4s}, [x8], #64
st1 {v4.4s,v5.4s,v6.4s,v7.4s}, [x8], #64
movrel x15, tbl_rev128_s
ld1 {v27.4s}, [x15]
.ifc \type, fixed
lsl x4, x4, #1
.else
lsl x4, x4, #2
.endif
add x10, x0, #45<<2
add x0, x0, #16<<2
add x1, x1, #16<<2
add x5, x3, x4, lsl #5
sub x5, x5, x4 // samples2
neg x13, x4 // -incr
mov x9, #64<<2
.ifc \type, fixed
ld1r {v16.2s}, [x2] // dither_state
sxtl v16.2d, v16.2s
movi v29.16b, #0
movi v30.2d, #(1<<OUT_SHIFT)-1
trn1 v31.2d, v29.2d, v30.2d
trn2 v30.2d, v30.2d, v29.2d
trn1 v16.2d, v16.2d, v29.2d
.else
movi v16.4s, #0
movi v28.4s, #0
.endif
mov x14, #4
1:
mov x8, x0
sub x7, x1, #3<<2
sub x6, x1, x14, lsl #4
add x7, x7, x14, lsl #4
add x11, x6, #(32)<<2 // w + 32
add x12, x7, #(32)<<2 // w2 + 32
mov x15, #8
movi v17.16b, #0
movi v18.16b, #0
movi v19.16b, #0
2:
subs x15, x15, #1
ld1 {v0.4s}, [x8], x9
ld1 {v1.4s}, [x10], x9
ld1 {v2.4s}, [x6], x9
ld1 {v3.4s}, [x7], x9
tbl v6.16b, {v0.16b}, v27.16b
tbl v7.16b, {v1.16b}, v27.16b
ld1 {v4.4s}, [x11], x9
ld1 {v5.4s}, [x12], x9
MLA v16, v2, v0
MLA2 v17, v2, v0
MLS v18, v3, v6
MLS2 v19, v3, v6
MLS v16, v4, v7
MLS2 v17, v4, v7
MLS v18, v5, v1
MLS2 v19, v5, v1
b.gt 2b
cmp x14, #4
sub x10, x10, #64<<5 // 64 * 8 * sizeof(int32_t)
.ifc \type, fixed
and v28.16b, v16.16b, v30.16b
ext v28.16b, v29.16b, v28.16b, #8
b.eq 4f
round_sample v19, 1, 1
4:
round_sample v16, 1, 0
shrn v16.2s, v16.2d, #OUT_SHIFT
round_sample v19, 0, 0
shrn v19.2s, v19.2d, #OUT_SHIFT
round_sample v17, 0, 1
round_sample v18, 1, 1
round_sample v17, 1, 0
shrn2 v16.4s, v17.2d, #OUT_SHIFT
round_sample v18, 0, 0
shrn2 v19.4s, v18.2d, #OUT_SHIFT
sqxtn v16.4h, v16.4s
sqxtn v18.4h, v19.4s
.else
ext v18.16b, v18.16b, v18.16b, #8
.endif
st1 {v16.\st\()}[0], [x3], x4
b.eq 4f
st1 {v18.\st\()}[1], [x5], x13
4:
st1 {v16.\st\()}[1], [x3], x4
st1 {v18.\st\()}[0], [x5], x13
st1 {v16.\st\()}[2], [x3], x4
st1 {v18.\st\()}[3], [x5], x13
st1 {v16.\st\()}[3], [x3], x4
st1 {v18.\st\()}[2], [x5], x13
mov v16.16b, v28.16b
subs x14, x14, #1
add x0, x0, #4<<2
sub x10, x10, #4<<2
b.gt 1b
// computing samples[16]
add x6, x1, #32<<2
ld1 {v0.2s}, [x6], x9
ld1 {v1.2s}, [x0], x9
.rept 3
ld1 {v2.2s}, [x6], x9
ld1 {v3.2s}, [x0], x9
MLS v16, v0, v1
ld1 {v0.2s}, [x6], x9
ld1 {v1.2s}, [x0], x9
MLS v16, v2, v3
.endr
ld1 {v2.2s}, [x6], x9
ld1 {v3.2s}, [x0], x9
MLS v16, v0, v1
MLS v16, v2, v3
.ifc \type, fixed
and v28.16b, v16.16b, v30.16b
shrn v20.2s, v16.2d, #OUT_SHIFT
xtn v28.2s, v28.2d
sqxtn v20.4h, v20.4s
st1 {v28.s}[0], [x2] // save dither_state
st1 {v20.h}[0], [x3]
.else
st1 {v16.s}[0], [x3]
.endif
ret
endfunc
.purgem round_sample
.purgem MLA
.purgem MLA2
.purgem MLS
.purgem MLS2
.endm
.macro round_sample r, idx, next
add \r\().2d, \r\().2d, v28.2d
.if \idx == 0
and v28.16b, \r\().16b, v30.16b
.else // \idx == 1
and v28.16b, \r\().16b, v31.16b
.endif
.if \idx != \next
.if \next == 0
ext v28.16b, v28.16b, v29.16b, #8
.else
ext v28.16b, v29.16b, v28.16b, #8
.endif
.endif
.endm
.macro MLA d, s1, s2
smlal \d\().2d, \s1\().2s, \s2\().2s
.endm
.macro MLA2 d, s1, s2
smlal2 \d\().2d, \s1\().4s, \s2\().4s
.endm
.macro MLS d, s1, s2
smlsl \d\().2d, \s1\().2s, \s2\().2s
.endm
.macro MLS2 d, s1, s2
smlsl2 \d\().2d, \s1\().4s, \s2\().4s
.endm
apply_window fixed, h
// nothing to do for round_sample and ML{A,S}2
.macro round_sample r, idx, next
.endm
.macro MLA2 d, s1, s2
.endm
.macro MLS2 d, s1, s2
.endm
.macro MLA d, s1, s2
fmla \d\().4s, \s1\().4s, \s2\().4s
.endm
.macro MLS d, s1, s2
fmls \d\().4s, \s1\().4s, \s2\().4s
.endm
apply_window float, s
@@ -0,0 +1,49 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stddef.h>
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/mpegvideoencdsp.h"
#include "config.h"
int ff_pix_sum16_neon(const uint8_t *pix, ptrdiff_t line_size);
int ff_pix_norm1_neon(const uint8_t *pix, ptrdiff_t line_size);
#if HAVE_DOTPROD
int ff_pix_norm1_neon_dotprod(const uint8_t *pix, ptrdiff_t line_size);
#endif
av_cold void ff_mpegvideoencdsp_init_aarch64(MpegvideoEncDSPContext *c,
AVCodecContext *avctx)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
c->pix_sum = ff_pix_sum16_neon;
c->pix_norm1 = ff_pix_norm1_neon;
}
#if HAVE_DOTPROD
if (have_dotprod(cpu_flags)) {
c->pix_norm1 = ff_pix_norm1_neon_dotprod;
}
#endif
}
@@ -0,0 +1,93 @@
/*
* Copyright (c) 2024 Ramiro Polla
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
function ff_pix_sum16_neon, export=1
// x0 const uint8_t *pix
// x1 ptrdiff_t line_size
add x2, x0, x1
lsl x1, x1, #1
movi v0.16b, #0
mov w3, #16
1:
ld1 {v1.16b}, [x0], x1
ld1 {v2.16b}, [x2], x1
subs w3, w3, #2
uadalp v0.8h, v1.16b
uadalp v0.8h, v2.16b
b.ne 1b
uaddlv s0, v0.8h
fmov w0, s0
ret
endfunc
function ff_pix_norm1_neon, export=1
// x0 const uint8_t *pix
// x1 ptrdiff_t line_size
movi v0.16b, #0
mov w2, #16
1:
ld1 {v1.16b}, [x0], x1
subs w2, w2, #1
umull v2.8h, v1.8b, v1.8b
umull2 v3.8h, v1.16b, v1.16b
uadalp v0.4s, v2.8h
uadalp v0.4s, v3.8h
b.ne 1b
uaddlv d0, v0.4s
fmov w0, s0
ret
endfunc
#if HAVE_DOTPROD
ENABLE_DOTPROD
function ff_pix_norm1_neon_dotprod, export=1
// x0 const uint8_t *pix
// x1 ptrdiff_t line_size
movi v0.16b, #0
mov w2, #16
1:
ld1 {v1.16b}, [x0], x1
ld1 {v2.16b}, [x0], x1
udot v0.4s, v1.16b, v1.16b
subs w2, w2, #2
udot v0.4s, v2.16b, v2.16b
b.ne 1b
uaddlv d0, v0.4s
fmov w0, s0
ret
endfunc
DISABLE_DOTPROD
#endif
+173
View File
@@ -0,0 +1,173 @@
/*
* This file is part of FFmpeg.
*
* Copyright (c) 2023 J. Dekker <jdek@itanimul.li>
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
.macro clip min, max, regs:vararg
.irp x, \regs
smax \x, \x, \min
.endr
.irp x, \regs
smin \x, \x, \max
.endr
.endm
.macro transpose_8x8B r0, r1, r2, r3, r4, r5, r6, r7, r8, r9
trn1 \r8\().8b, \r0\().8b, \r1\().8b
trn2 \r9\().8b, \r0\().8b, \r1\().8b
trn1 \r1\().8b, \r2\().8b, \r3\().8b
trn2 \r3\().8b, \r2\().8b, \r3\().8b
trn1 \r0\().8b, \r4\().8b, \r5\().8b
trn2 \r5\().8b, \r4\().8b, \r5\().8b
trn1 \r2\().8b, \r6\().8b, \r7\().8b
trn2 \r7\().8b, \r6\().8b, \r7\().8b
trn1 \r4\().4h, \r0\().4h, \r2\().4h
trn2 \r2\().4h, \r0\().4h, \r2\().4h
trn1 \r6\().4h, \r5\().4h, \r7\().4h
trn2 \r7\().4h, \r5\().4h, \r7\().4h
trn1 \r5\().4h, \r9\().4h, \r3\().4h
trn2 \r9\().4h, \r9\().4h, \r3\().4h
trn1 \r3\().4h, \r8\().4h, \r1\().4h
trn2 \r8\().4h, \r8\().4h, \r1\().4h
trn1 \r0\().2s, \r3\().2s, \r4\().2s
trn2 \r4\().2s, \r3\().2s, \r4\().2s
trn1 \r1\().2s, \r5\().2s, \r6\().2s
trn2 \r5\().2s, \r5\().2s, \r6\().2s
trn2 \r6\().2s, \r8\().2s, \r2\().2s
trn1 \r2\().2s, \r8\().2s, \r2\().2s
trn1 \r3\().2s, \r9\().2s, \r7\().2s
trn2 \r7\().2s, \r9\().2s, \r7\().2s
.endm
.macro transpose_8x16B r0, r1, r2, r3, r4, r5, r6, r7, t0, t1
trn1 \t0\().16b, \r0\().16b, \r1\().16b
trn2 \t1\().16b, \r0\().16b, \r1\().16b
trn1 \r1\().16b, \r2\().16b, \r3\().16b
trn2 \r3\().16b, \r2\().16b, \r3\().16b
trn1 \r0\().16b, \r4\().16b, \r5\().16b
trn2 \r5\().16b, \r4\().16b, \r5\().16b
trn1 \r2\().16b, \r6\().16b, \r7\().16b
trn2 \r7\().16b, \r6\().16b, \r7\().16b
trn1 \r4\().8h, \r0\().8h, \r2\().8h
trn2 \r2\().8h, \r0\().8h, \r2\().8h
trn1 \r6\().8h, \r5\().8h, \r7\().8h
trn2 \r7\().8h, \r5\().8h, \r7\().8h
trn1 \r5\().8h, \t1\().8h, \r3\().8h
trn2 \t1\().8h, \t1\().8h, \r3\().8h
trn1 \r3\().8h, \t0\().8h, \r1\().8h
trn2 \t0\().8h, \t0\().8h, \r1\().8h
trn1 \r0\().4s, \r3\().4s, \r4\().4s
trn2 \r4\().4s, \r3\().4s, \r4\().4s
trn1 \r1\().4s, \r5\().4s, \r6\().4s
trn2 \r5\().4s, \r5\().4s, \r6\().4s
trn2 \r6\().4s, \t0\().4s, \r2\().4s
trn1 \r2\().4s, \t0\().4s, \r2\().4s
trn1 \r3\().4s, \t1\().4s, \r7\().4s
trn2 \r7\().4s, \t1\().4s, \r7\().4s
.endm
.macro transpose_4x16B r0, r1, r2, r3, t4, t5, t6, t7
trn1 \t4\().16b, \r0\().16b, \r1\().16b
trn2 \t5\().16b, \r0\().16b, \r1\().16b
trn1 \t6\().16b, \r2\().16b, \r3\().16b
trn2 \t7\().16b, \r2\().16b, \r3\().16b
trn1 \r0\().8h, \t4\().8h, \t6\().8h
trn2 \r2\().8h, \t4\().8h, \t6\().8h
trn1 \r1\().8h, \t5\().8h, \t7\().8h
trn2 \r3\().8h, \t5\().8h, \t7\().8h
.endm
.macro transpose_4x8B r0, r1, r2, r3, t4, t5, t6, t7
trn1 \t4\().8b, \r0\().8b, \r1\().8b
trn2 \t5\().8b, \r0\().8b, \r1\().8b
trn1 \t6\().8b, \r2\().8b, \r3\().8b
trn2 \t7\().8b, \r2\().8b, \r3\().8b
trn1 \r0\().4h, \t4\().4h, \t6\().4h
trn2 \r2\().4h, \t4\().4h, \t6\().4h
trn1 \r1\().4h, \t5\().4h, \t7\().4h
trn2 \r3\().4h, \t5\().4h, \t7\().4h
.endm
.macro transpose_4x4H r0, r1, r2, r3, r4, r5, r6, r7
trn1 \r4\().4h, \r0\().4h, \r1\().4h
trn2 \r5\().4h, \r0\().4h, \r1\().4h
trn1 \r6\().4h, \r2\().4h, \r3\().4h
trn2 \r7\().4h, \r2\().4h, \r3\().4h
trn1 \r0\().2s, \r4\().2s, \r6\().2s
trn2 \r2\().2s, \r4\().2s, \r6\().2s
trn1 \r1\().2s, \r5\().2s, \r7\().2s
trn2 \r3\().2s, \r5\().2s, \r7\().2s
.endm
.macro transpose_4x8H r0, r1, r2, r3, t4, t5, t6, t7
trn1 \t4\().8h, \r0\().8h, \r1\().8h
trn2 \t5\().8h, \r0\().8h, \r1\().8h
trn1 \t6\().8h, \r2\().8h, \r3\().8h
trn2 \t7\().8h, \r2\().8h, \r3\().8h
trn1 \r0\().4s, \t4\().4s, \t6\().4s
trn2 \r2\().4s, \t4\().4s, \t6\().4s
trn1 \r1\().4s, \t5\().4s, \t7\().4s
trn2 \r3\().4s, \t5\().4s, \t7\().4s
.endm
.macro transpose_8x8H r0, r1, r2, r3, r4, r5, r6, r7, r8, r9
trn1 \r8\().8h, \r0\().8h, \r1\().8h
trn2 \r9\().8h, \r0\().8h, \r1\().8h
trn1 \r1\().8h, \r2\().8h, \r3\().8h
trn2 \r3\().8h, \r2\().8h, \r3\().8h
trn1 \r0\().8h, \r4\().8h, \r5\().8h
trn2 \r5\().8h, \r4\().8h, \r5\().8h
trn1 \r2\().8h, \r6\().8h, \r7\().8h
trn2 \r7\().8h, \r6\().8h, \r7\().8h
trn1 \r4\().4s, \r0\().4s, \r2\().4s
trn2 \r2\().4s, \r0\().4s, \r2\().4s
trn1 \r6\().4s, \r5\().4s, \r7\().4s
trn2 \r7\().4s, \r5\().4s, \r7\().4s
trn1 \r5\().4s, \r9\().4s, \r3\().4s
trn2 \r9\().4s, \r9\().4s, \r3\().4s
trn1 \r3\().4s, \r8\().4s, \r1\().4s
trn2 \r8\().4s, \r8\().4s, \r1\().4s
trn1 \r0\().2d, \r3\().2d, \r4\().2d
trn2 \r4\().2d, \r3\().2d, \r4\().2d
trn1 \r1\().2d, \r5\().2d, \r6\().2d
trn2 \r5\().2d, \r5\().2d, \r6\().2d
trn2 \r6\().2d, \r8\().2d, \r2\().2d
trn1 \r2\().2d, \r8\().2d, \r2\().2d
trn1 \r3\().2d, \r9\().2d, \r7\().2d
trn2 \r7\().2d, \r9\().2d, \r7\().2d
.endm
+66
View File
@@ -0,0 +1,66 @@
/*
* check NEON registers for clobbers
* Copyright (c) 2013 Martin Storsjo
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavcodec/avcodec.h"
#include "libavutil/aarch64/neontest.h"
wrap(avcodec_open2(AVCodecContext *avctx,
const AVCodec *codec,
AVDictionary **options))
{
testneonclobbers(avcodec_open2, avctx, codec, options);
}
wrap(avcodec_decode_subtitle2(AVCodecContext *avctx,
AVSubtitle *sub,
int *got_sub_ptr,
AVPacket *avpkt))
{
testneonclobbers(avcodec_decode_subtitle2, avctx, sub,
got_sub_ptr, avpkt);
}
wrap(avcodec_encode_subtitle(AVCodecContext *avctx,
uint8_t *buf, int buf_size,
const AVSubtitle *sub))
{
testneonclobbers(avcodec_encode_subtitle, avctx, buf, buf_size, sub);
}
wrap(avcodec_send_packet(AVCodecContext *avctx, const AVPacket *avpkt))
{
testneonclobbers(avcodec_send_packet, avctx, avpkt);
}
wrap(avcodec_receive_packet(AVCodecContext *avctx, AVPacket *avpkt))
{
testneonclobbers(avcodec_receive_packet, avctx, avpkt);
}
wrap(avcodec_send_frame(AVCodecContext *avctx, const AVFrame *frame))
{
testneonclobbers(avcodec_send_frame, avctx, frame);
}
wrap(avcodec_receive_frame(AVCodecContext *avctx, AVFrame *frame))
{
testneonclobbers(avcodec_receive_frame, avctx, frame);
}
+36
View File
@@ -0,0 +1,36 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/opus/dsp.h"
void ff_opus_postfilter_neon(float *data, int period, float *gains, int len);
float ff_opus_deemphasis_neon(float *out, float *in, float coeff, const float *weights, int len);
av_cold void ff_opus_dsp_init_aarch64(OpusDSP *ctx)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
ctx->postfilter = ff_opus_postfilter_neon;
ctx->deemphasis = ff_opus_deemphasis_neon;
}
}
+95
View File
@@ -0,0 +1,95 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
function ff_opus_deemphasis_neon, export=1
ld1 {v4.4s}, [x2], #16
ld1 {v5.4s}, [x2], #16
ld1 {v6.4s}, [x2], #16
ld1 {v7.4s}, [x2]
fmul v0.4s, v4.4s, v0.s[0]
1: ld1 {v1.4s, v2.4s}, [x1], #32
fmla v0.4s, v5.4s, v1.s[0]
fmul v3.4s, v7.4s, v2.s[2]
fmla v0.4s, v6.4s, v1.s[1]
fmla v3.4s, v6.4s, v2.s[1]
fmla v0.4s, v7.4s, v1.s[2]
fmla v3.4s, v5.4s, v2.s[0]
fadd v1.4s, v1.4s, v0.4s
fadd v2.4s, v2.4s, v3.4s
fmla v2.4s, v4.4s, v1.s[3]
st1 {v1.4s, v2.4s}, [x0], #32
fmul v0.4s, v4.4s, v2.s[3]
subs w3, w3, #8
b.gt 1b
mov s0, v2.s[3]
ret
endfunc
function ff_opus_postfilter_neon, export=1
ld1 {v0.4s}, [x2]
dup v1.4s, v0.s[1]
dup v2.4s, v0.s[2]
dup v0.4s, v0.s[0]
add w1, w1, #2
sub x1, x0, x1, lsl #2
ld1 {v3.4s}, [x1]
fmul v3.4s, v3.4s, v2.4s
1: add x1, x1, #4
ld1 {v4.4s}, [x1]
add x1, x1, #4
ld1 {v5.4s}, [x1]
add x1, x1, #4
ld1 {v6.4s}, [x1]
add x1, x1, #4
ld1 {v7.4s}, [x1]
fmla v3.4s, v7.4s, v2.4s
fadd v6.4s, v6.4s, v4.4s
ld1 {v4.4s}, [x0]
fmla v4.4s, v5.4s, v0.4s
fmul v6.4s, v6.4s, v1.4s
fadd v6.4s, v6.4s, v3.4s
fadd v4.4s, v4.4s, v6.4s
fmul v3.4s, v7.4s, v2.4s
st1 {v4.4s}, [x0], #16
subs w3, w3, #4
b.gt 1b
ret
endfunc
@@ -0,0 +1,46 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/pixblockdsp.h"
void ff_get_pixels_neon(int16_t *block, const uint8_t *pixels,
ptrdiff_t stride);
void ff_diff_pixels_neon(int16_t *block, const uint8_t *s1,
const uint8_t *s2, ptrdiff_t stride);
av_cold void ff_pixblockdsp_init_aarch64(PixblockDSPContext *c,
AVCodecContext *avctx,
unsigned high_bit_depth)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
if (!high_bit_depth) {
c->get_pixels_unaligned =
c->get_pixels = ff_get_pixels_neon;
}
c->diff_pixels_unaligned =
c->diff_pixels = ff_diff_pixels_neon;
}
}
@@ -0,0 +1,51 @@
/*
* Copyright (c) 2020 Martin Storsjo
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
function ff_get_pixels_neon, export=1
mov w3, #8
1:
ld1 {v0.8b}, [x1], x2
subs w3, w3, #2
ld1 {v1.8b}, [x1], x2
uxtl v0.8h, v0.8b
uxtl v1.8h, v1.8b
st1 {v0.8h, v1.8h}, [x0], #32
b.gt 1b
ret
endfunc
function ff_diff_pixels_neon, export=1
mov w4, #8
1:
ld1 {v0.8b}, [x1], x3
ld1 {v1.8b}, [x2], x3
subs w4, w4, #2
ld1 {v2.8b}, [x1], x3
usubl v0.8h, v0.8b, v1.8b
ld1 {v3.8b}, [x2], x3
usubl v1.8h, v2.8b, v3.8b
st1 {v0.8h, v1.8h}, [x0], #32
b.gt 1b
ret
endfunc
@@ -0,0 +1,48 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/rv34dsp.h"
#include "config.h"
void ff_put_rv40_chroma_mc8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_put_rv40_chroma_mc4_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_rv40_chroma_mc8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_rv40_chroma_mc4_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
av_cold void ff_rv40dsp_init_aarch64(RV34DSPContext *c)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
c->put_chroma_pixels_tab[0] = ff_put_rv40_chroma_mc8_neon;
c->put_chroma_pixels_tab[1] = ff_put_rv40_chroma_mc4_neon;
c->avg_chroma_pixels_tab[0] = ff_avg_rv40_chroma_mc8_neon;
c->avg_chroma_pixels_tab[1] = ff_avg_rv40_chroma_mc4_neon;
}
}
@@ -0,0 +1,70 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/aarch64/cpu.h"
#include "libavutil/attributes.h"
#include "libavcodec/sbrdsp.h"
void ff_sbr_sum64x5_neon(float *z);
float ff_sbr_sum_square_neon(float (*x)[2], int n);
void ff_sbr_neg_odd_64_neon(float *x);
void ff_sbr_qmf_pre_shuffle_neon(float *z);
void ff_sbr_qmf_post_shuffle_neon(float W[32][2], const float *z);
void ff_sbr_qmf_deint_neg_neon(float *v, const float *src);
void ff_sbr_qmf_deint_bfly_neon(float *v, const float *src0, const float *src1);
void ff_sbr_hf_g_filt_neon(float (*Y)[2], const float (*X_high)[40][2],
const float *g_filt, int m_max, intptr_t ixh);
void ff_sbr_hf_gen_neon(float (*X_high)[2], const float (*X_low)[2],
const float alpha0[2], const float alpha1[2],
float bw, int start, int end);
void ff_sbr_autocorrelate_neon(const float x[40][2], float phi[3][2][2]);
void ff_sbr_hf_apply_noise_0_neon(float Y[64][2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_hf_apply_noise_1_neon(float Y[64][2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_hf_apply_noise_2_neon(float Y[64][2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
void ff_sbr_hf_apply_noise_3_neon(float Y[64][2], const float *s_m,
const float *q_filt, int noise,
int kx, int m_max);
av_cold void ff_sbrdsp_init_aarch64(SBRDSPContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
s->sum64x5 = ff_sbr_sum64x5_neon;
s->sum_square = ff_sbr_sum_square_neon;
s->neg_odd_64 = ff_sbr_neg_odd_64_neon;
s->qmf_pre_shuffle = ff_sbr_qmf_pre_shuffle_neon;
s->qmf_post_shuffle = ff_sbr_qmf_post_shuffle_neon;
s->qmf_deint_neg = ff_sbr_qmf_deint_neg_neon;
s->qmf_deint_bfly = ff_sbr_qmf_deint_bfly_neon;
s->hf_g_filt = ff_sbr_hf_g_filt_neon;
s->hf_gen = ff_sbr_hf_gen_neon;
s->autocorrelate = ff_sbr_autocorrelate_neon;
s->hf_apply_noise[0] = ff_sbr_hf_apply_noise_0_neon;
s->hf_apply_noise[1] = ff_sbr_hf_apply_noise_1_neon;
s->hf_apply_noise[2] = ff_sbr_hf_apply_noise_2_neon;
s->hf_apply_noise[3] = ff_sbr_hf_apply_noise_3_neon;
}
}
+327
View File
@@ -0,0 +1,327 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
const factors, align=4
.float 1.0, -1.0, 1.0, -1.0
endconst
const phi_noise_0, align=4
.float 1.0, 0.0, 1.0, 0.0
endconst
const phi_noise_1, align=4
.float 0.0, 1.0, 0.0, -1.0
.float 0.0, -1.0, 0.0, 1.0
endconst
const phi_noise_2, align=4
.float -1.0, 0.0, -1.0, 0.0
endconst
const phi_noise_3, align=4
.float 0.0, -1.0, 0.0, 1.0
.float 0.0, 1.0, 0.0, -1.0
endconst
function ff_sbr_sum64x5_neon, export=1
add x1, x0, #64*4
add x2, x0, #128*4
add x3, x0, #192*4
add x4, x0, #256*4
mov x5, #64
1: ld1 {v0.4s}, [x0]
ld1 {v1.4s}, [x1], #16
fadd v0.4s, v0.4s, v1.4s
ld1 {v2.4s}, [x2], #16
fadd v0.4s, v0.4s, v2.4s
ld1 {v3.4s}, [x3], #16
fadd v0.4s, v0.4s, v3.4s
ld1 {v4.4s}, [x4], #16
fadd v0.4s, v0.4s, v4.4s
st1 {v0.4s}, [x0], #16
subs x5, x5, #4
b.gt 1b
ret
endfunc
function ff_sbr_sum_square_neon, export=1
movi v0.4s, #0
1: ld1 {v1.4s}, [x0], #16
fmla v0.4s, v1.4s, v1.4s
subs w1, w1, #2
b.gt 1b
faddp v0.4s, v0.4s, v0.4s
faddp v0.4s, v0.4s, v0.4s
ret
endfunc
function ff_sbr_neg_odd_64_neon, export=1
mov x1, x0
movi v5.4s, #1<<7, lsl #24
ld2 {v0.4s, v1.4s}, [x0], #32
eor v1.16b, v1.16b, v5.16b
ld2 {v2.4s, v3.4s}, [x0], #32
.rept 3
st2 {v0.4s, v1.4s}, [x1], #32
eor v3.16b, v3.16b, v5.16b
ld2 {v0.4s, v1.4s}, [x0], #32
st2 {v2.4s, v3.4s}, [x1], #32
eor v1.16b, v1.16b, v5.16b
ld2 {v2.4s, v3.4s}, [x0], #32
.endr
eor v3.16b, v3.16b, v5.16b
st2 {v0.4s, v1.4s}, [x1], #32
st2 {v2.4s, v3.4s}, [x1], #32
ret
endfunc
function ff_sbr_qmf_pre_shuffle_neon, export=1
add x1, x0, #60*4
add x2, x0, #64*4
mov x3, #-16
mov x4, #-4
movi v6.4s, #1<<7, lsl #24
ld1 {v0.2s}, [x0], #8
st1 {v0.2s}, [x2], #8
.rept 7
ld1 {v1.4s}, [x1], x3
ld1 {v2.4s}, [x0], #16
eor v1.16b, v1.16b, v6.16b
rev64 v1.4s, v1.4s
ext v1.16b, v1.16b, v1.16b, #8
st2 {v1.4s, v2.4s}, [x2], #32
.endr
add x1, x1, #8
ld1 {v1.2s}, [x1], x4
ld1 {v2.2s}, [x0], #8
ld1 {v1.s}[3], [x1]
ld1 {v2.s}[2], [x0]
eor v1.16b, v1.16b, v6.16b
rev64 v1.4s, v1.4s
st2 {v1.2s, v2.2s}, [x2], #16
st2 {v1.s, v2.s}[2], [x2]
ret
endfunc
function ff_sbr_qmf_post_shuffle_neon, export=1
add x2, x1, #60*4
mov x3, #-16
mov x4, #32
movi v6.4s, #1<<7, lsl #24
1: ld1 {v0.4s}, [x2], x3
ld1 {v1.4s}, [x1], #16
eor v0.16b, v0.16b, v6.16b
rev64 v0.4s, v0.4s
ext v0.16b, v0.16b, v0.16b, #8
st2 {v0.4s, v1.4s}, [x0], #32
subs x4, x4, #4
b.gt 1b
ret
endfunc
function ff_sbr_qmf_deint_neg_neon, export=1
add x1, x1, #56*4
add x2, x0, #60*4
mov x3, #-32
mov x4, #32
movi v2.4s, #1<<7, lsl #24
1: ld2 {v0.4s, v1.4s}, [x1], x3
eor v0.16b, v0.16b, v2.16b
rev64 v1.4s, v1.4s
ext v1.16b, v1.16b, v1.16b, #8
st1 {v0.4s}, [x2]
st1 {v1.4s}, [x0], #16
sub x2, x2, #16
subs x4, x4, #4
b.gt 1b
ret
endfunc
function ff_sbr_qmf_deint_bfly_neon, export=1
add x2, x2, #60*4
add x3, x0, #124*4
mov x4, #64
mov x5, #-16
1: ld1 {v0.4s}, [x1], #16
ld1 {v1.4s}, [x2], x5
rev64 v2.4s, v0.4s
ext v2.16b, v2.16b, v2.16b, #8
rev64 v3.4s, v1.4s
ext v3.16b, v3.16b, v3.16b, #8
fadd v1.4s, v1.4s, v2.4s
fsub v0.4s, v0.4s, v3.4s
st1 {v0.4s}, [x0], #16
st1 {v1.4s}, [x3], x5
subs x4, x4, #4
b.gt 1b
ret
endfunc
function ff_sbr_hf_gen_neon, export=1
sxtw x4, w4
sxtw x5, w5
movrel x6, factors
ld1 {v7.4s}, [x6]
dup v1.4s, v0.s[0]
mov v2.8b, v1.8b
mov v2.s[2], v7.s[0]
mov v2.s[3], v7.s[0]
fmul v1.4s, v1.4s, v2.4s
ld1 {v0.d}[0], [x3]
ld1 {v0.d}[1], [x2]
fmul v0.4s, v0.4s, v1.4s
fmul v1.4s, v0.4s, v7.4s
rev64 v0.4s, v0.4s
sub x7, x5, x4
add x0, x0, x4, lsl #3
add x1, x1, x4, lsl #3
sub x1, x1, #16
1: ld1 {v2.4s}, [x1], #16
ld1 {v3.2s}, [x1]
fmul v4.4s, v2.4s, v1.4s
fmul v5.4s, v2.4s, v0.4s
faddp v4.4s, v4.4s, v4.4s
faddp v5.4s, v5.4s, v5.4s
faddp v4.4s, v4.4s, v4.4s
faddp v5.4s, v5.4s, v5.4s
mov v4.s[1], v5.s[0]
fadd v4.2s, v4.2s, v3.2s
st1 {v4.2s}, [x0], #8
sub x1, x1, #8
subs x7, x7, #1
b.gt 1b
ret
endfunc
function ff_sbr_hf_g_filt_neon, export=1
sxtw x3, w3
sxtw x4, w4
mov x5, #40*2*4
add x1, x1, x4, lsl #3
1: ld1 {v0.2s}, [x1], x5
ld1 {v1.s}[0], [x2], #4
fmul v2.4s, v0.4s, v1.s[0]
st1 {v2.2s}, [x0], #8
subs x3, x3, #1
b.gt 1b
ret
endfunc
function ff_sbr_autocorrelate_neon, export=1
mov x2, #38
movrel x3, factors
ld1 {v0.4s}, [x3]
movi v1.4s, #0
movi v2.4s, #0
movi v3.4s, #0
ld1 {v4.2s}, [x0], #8
ld1 {v5.2s}, [x0], #8
fmul v16.2s, v4.2s, v4.2s
fmul v17.2s, v5.2s, v4.s[0]
fmul v18.2s, v5.2s, v4.s[1]
1: ld1 {v5.d}[1], [x0], #8
fmla v1.2s, v4.2s, v4.2s
fmla v2.4s, v5.4s, v4.s[0]
fmla v3.4s, v5.4s, v4.s[1]
mov v4.d[0], v5.d[0]
mov v5.d[0], v5.d[1]
subs x2, x2, #1
b.gt 1b
fmul v19.2s, v4.2s, v4.2s
fmul v20.2s, v5.2s, v4.s[0]
fmul v21.2s, v5.2s, v4.s[1]
fadd v22.4s, v2.4s, v20.4s
fsub v22.4s, v22.4s, v17.4s
fadd v23.4s, v3.4s, v21.4s
fsub v23.4s, v23.4s, v18.4s
rev64 v23.4s, v23.4s
fmul v23.4s, v23.4s, v0.4s
fadd v22.4s, v22.4s, v23.4s
st1 {v22.4s}, [x1], #16
fadd v23.2s, v1.2s, v19.2s
fsub v23.2s, v23.2s, v16.2s
faddp v23.2s, v23.2s, v23.2s
st1 {v23.s}[0], [x1]
add x1, x1, #8
rev64 v3.2s, v3.2s
fmul v3.2s, v3.2s, v0.2s
fadd v2.2s, v2.2s, v3.2s
st1 {v2.2s}, [x1]
add x1, x1, #16
faddp v1.2s, v1.2s, v1.2s
st1 {v1.s}[0], [x1]
ret
endfunc
.macro apply_noise_common
sxtw x3, w3
sxtw x5, w5
movrel x7, X(ff_sbr_noise_table)
add x3, x3, #1
1: and x3, x3, #0x1ff
add x8, x7, x3, lsl #3
add x3, x3, #2
ld1 {v2.4s}, [x0]
ld1 {v3.2s}, [x1], #8
ld1 {v4.2s}, [x2], #8
ld1 {v5.4s}, [x8]
mov v6.16b, v2.16b
zip1 v3.4s, v3.4s, v3.4s
zip1 v4.4s, v4.4s, v4.4s
fmla v6.4s, v1.4s, v3.4s
fmla v2.4s, v5.4s, v4.4s
fcmeq v7.4s, v3.4s, #0
bif v2.16b, v6.16b, v7.16b
st1 {v2.4s}, [x0], #16
subs x5, x5, #2
b.gt 1b
.endm
function ff_sbr_hf_apply_noise_0_neon, export=1
movrel x9, phi_noise_0
ld1 {v1.4s}, [x9]
apply_noise_common
ret
endfunc
function ff_sbr_hf_apply_noise_1_neon, export=1
movrel x9, phi_noise_1
and x4, x4, #1
add x9, x9, x4, lsl #4
ld1 {v1.4s}, [x9]
apply_noise_common
ret
endfunc
function ff_sbr_hf_apply_noise_2_neon, export=1
movrel x9, phi_noise_2
ld1 {v1.4s}, [x9]
apply_noise_common
ret
endfunc
function ff_sbr_hf_apply_noise_3_neon, export=1
movrel x9, phi_noise_3
and x4, x4, #1
add x9, x9, x4, lsl #4
ld1 {v1.4s}, [x9]
apply_noise_common
ret
endfunc
@@ -0,0 +1,362 @@
/*
* ARM NEON IDCT
*
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2017 Matthieu Bouron <matthieu.bouron@gmail.com>
*
* Based on Simple IDCT
* Copyright (c) 2001 Michael Niedermayer <michaelni@gmx.at>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#define Z1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z6 8867 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z7 4520 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
#define Z4c ((1<<(COL_SHIFT-1))/Z4)
#define ROW_SHIFT 11
#define COL_SHIFT 20
#define z1 v0.H[0]
#define z2 v0.H[1]
#define z3 v0.H[2]
#define z4 v0.H[3]
#define z5 v0.H[4]
#define z6 v0.H[5]
#define z7 v0.H[6]
#define z4c v0.H[7]
const idct_coeff_neon, align=4
.short Z1, Z2, Z3, Z4, Z5, Z6, Z7, Z4c
endconst
.macro idct_start data
prfm pldl1keep, [\data]
mov x10, x30
movrel x3, idct_coeff_neon
ld1 {v0.2d}, [x3]
.endm
.macro idct_end
ret x10
.endm
.macro smull1 a, b, c
smull \a, \b, \c
.endm
.macro smlal1 a, b, c
smlal \a, \b, \c
.endm
.macro smlsl1 a, b, c
smlsl \a, \b, \c
.endm
.macro idct_col4_top y1, y2, y3, y4, i, l
smull\i v7.4s, \y3\l, z2
smull\i v16.4s, \y3\l, z6
smull\i v17.4s, \y2\l, z1
add v19.4s, v23.4s, v7.4s
smull\i v18.4s, \y2\l, z3
add v20.4s, v23.4s, v16.4s
smull\i v5.4s, \y2\l, z5
sub v21.4s, v23.4s, v16.4s
smull\i v6.4s, \y2\l, z7
sub v22.4s, v23.4s, v7.4s
smlal\i v17.4s, \y4\l, z3
smlsl\i v18.4s, \y4\l, z7
smlsl\i v5.4s, \y4\l, z1
smlsl\i v6.4s, \y4\l, z5
.endm
.macro idct_row4_neon y1, y2, y3, y4, pass
ld1 {\y1\().2d,\y2\().2d}, [x2], #32
movi v23.4s, #1<<2, lsl #8
orr v5.16b, \y1\().16b, \y2\().16b
ld1 {\y3\().2d,\y4\().2d}, [x2], #32
orr v6.16b, \y3\().16b, \y4\().16b
orr v5.16b, v5.16b, v6.16b
mov x3, v5.d[1]
smlal v23.4s, \y1\().4h, z4
idct_col4_top \y1, \y2, \y3, \y4, 1, .4h
cmp x3, #0
b.eq \pass\()f
smull2 v7.4s, \y1\().8h, z4
smlal2 v17.4s, \y2\().8h, z5
smlsl2 v18.4s, \y2\().8h, z1
smull2 v16.4s, \y3\().8h, z2
smlal2 v5.4s, \y2\().8h, z7
add v19.4s, v19.4s, v7.4s
sub v20.4s, v20.4s, v7.4s
sub v21.4s, v21.4s, v7.4s
add v22.4s, v22.4s, v7.4s
smlal2 v6.4s, \y2\().8h, z3
smull2 v7.4s, \y3\().8h, z6
smlal2 v17.4s, \y4\().8h, z7
smlsl2 v18.4s, \y4\().8h, z5
smlal2 v5.4s, \y4\().8h, z3
smlsl2 v6.4s, \y4\().8h, z1
add v19.4s, v19.4s, v7.4s
sub v20.4s, v20.4s, v16.4s
add v21.4s, v21.4s, v16.4s
sub v22.4s, v22.4s, v7.4s
\pass: add \y3\().4S, v19.4S, v17.4S
add \y4\().4s, v20.4s, v18.4s
shrn \y1\().4h, \y3\().4s, #ROW_SHIFT
shrn \y2\().4h, \y4\().4s, #ROW_SHIFT
add v7.4s, v21.4s, v5.4s
add v16.4s, v22.4s, v6.4s
shrn \y3\().4h, v7.4s, #ROW_SHIFT
shrn \y4\().4h, v16.4s, #ROW_SHIFT
sub v22.4s, v22.4s, v6.4s
sub v19.4s, v19.4s, v17.4s
sub v21.4s, v21.4s, v5.4s
shrn2 \y1\().8h, v22.4s, #ROW_SHIFT
sub v20.4s, v20.4s, v18.4s
shrn2 \y2\().8h, v21.4s, #ROW_SHIFT
shrn2 \y3\().8h, v20.4s, #ROW_SHIFT
shrn2 \y4\().8h, v19.4s, #ROW_SHIFT
trn1 v16.8h, \y1\().8h, \y2\().8h
trn2 v17.8h, \y1\().8h, \y2\().8h
trn1 v18.8h, \y3\().8h, \y4\().8h
trn2 v19.8h, \y3\().8h, \y4\().8h
trn1 \y1\().4s, v16.4s, v18.4s
trn1 \y2\().4s, v17.4s, v19.4s
trn2 \y3\().4s, v16.4s, v18.4s
trn2 \y4\().4s, v17.4s, v19.4s
.endm
.macro declare_idct_col4_neon i, l
function idct_col4_neon\i
dup v23.4h, z4c
.if \i == 1
add v23.4h, v23.4h, v24.4h
.else
mov v5.d[0], v24.d[1]
add v23.4h, v23.4h, v5.4h
.endif
smull v23.4s, v23.4h, z4
idct_col4_top v24, v25, v26, v27, \i, \l
mov x4, v28.d[\i - 1]
mov x5, v29.d[\i - 1]
cmp x4, #0
b.eq 1f
smull\i v7.4s, v28\l, z4
add v19.4s, v19.4s, v7.4s
sub v20.4s, v20.4s, v7.4s
sub v21.4s, v21.4s, v7.4s
add v22.4s, v22.4s, v7.4s
1: mov x4, v30.d[\i - 1]
cmp x5, #0
b.eq 2f
smlal\i v17.4s, v29\l, z5
smlsl\i v18.4s, v29\l, z1
smlal\i v5.4s, v29\l, z7
smlal\i v6.4s, v29\l, z3
2: mov x5, v31.d[\i - 1]
cmp x4, #0
b.eq 3f
smull\i v7.4s, v30\l, z6
smull\i v16.4s, v30\l, z2
add v19.4s, v19.4s, v7.4s
sub v22.4s, v22.4s, v7.4s
sub v20.4s, v20.4s, v16.4s
add v21.4s, v21.4s, v16.4s
3: cmp x5, #0
b.eq 4f
smlal\i v17.4s, v31\l, z7
smlsl\i v18.4s, v31\l, z5
smlal\i v5.4s, v31\l, z3
smlsl\i v6.4s, v31\l, z1
4: addhn v7.4h, v19.4s, v17.4s
addhn2 v7.8h, v20.4s, v18.4s
subhn v18.4h, v20.4s, v18.4s
subhn2 v18.8h, v19.4s, v17.4s
addhn v16.4h, v21.4s, v5.4s
addhn2 v16.8h, v22.4s, v6.4s
subhn v17.4h, v22.4s, v6.4s
subhn2 v17.8h, v21.4s, v5.4s
ret
endfunc
.endm
declare_idct_col4_neon 1, .4H
declare_idct_col4_neon 2, .8H
function ff_simple_idct_put_neon, export=1
idct_start x2
idct_row4_neon v24, v25, v26, v27, 1
idct_row4_neon v28, v29, v30, v31, 2
bl idct_col4_neon1
sqshrun v1.8b, v7.8h, #COL_SHIFT-16
sqshrun2 v1.16b, v16.8h, #COL_SHIFT-16
sqshrun v3.8b, v17.8h, #COL_SHIFT-16
sqshrun2 v3.16b, v18.8h, #COL_SHIFT-16
bl idct_col4_neon2
sqshrun v2.8b, v7.8h, #COL_SHIFT-16
sqshrun2 v2.16b, v16.8h, #COL_SHIFT-16
sqshrun v4.8b, v17.8h, #COL_SHIFT-16
sqshrun2 v4.16b, v18.8h, #COL_SHIFT-16
zip1 v16.4s, v1.4s, v2.4s
zip2 v17.4s, v1.4s, v2.4s
st1 {v16.d}[0], [x0], x1
st1 {v16.d}[1], [x0], x1
zip1 v18.4s, v3.4s, v4.4s
zip2 v19.4s, v3.4s, v4.4s
st1 {v17.d}[0], [x0], x1
st1 {v17.d}[1], [x0], x1
st1 {v18.d}[0], [x0], x1
st1 {v18.d}[1], [x0], x1
st1 {v19.d}[0], [x0], x1
st1 {v19.d}[1], [x0], x1
idct_end
endfunc
function ff_simple_idct_add_neon, export=1
idct_start x2
idct_row4_neon v24, v25, v26, v27, 1
idct_row4_neon v28, v29, v30, v31, 2
bl idct_col4_neon1
sshr v1.8h, v7.8h, #COL_SHIFT-16
sshr v2.8h, v16.8h, #COL_SHIFT-16
sshr v3.8h, v17.8h, #COL_SHIFT-16
sshr v4.8h, v18.8h, #COL_SHIFT-16
bl idct_col4_neon2
sshr v7.8h, v7.8h, #COL_SHIFT-16
sshr v16.8h, v16.8h, #COL_SHIFT-16
sshr v17.8h, v17.8h, #COL_SHIFT-16
sshr v18.8h, v18.8h, #COL_SHIFT-16
mov x9, x0
ld1 {v19.d}[0], [x0], x1
zip1 v23.2d, v1.2d, v7.2d
zip2 v24.2d, v1.2d, v7.2d
ld1 {v19.d}[1], [x0], x1
zip1 v25.2d, v2.2d, v16.2d
zip2 v26.2d, v2.2d, v16.2d
ld1 {v20.d}[0], [x0], x1
zip1 v27.2d, v3.2d, v17.2d
zip2 v28.2d, v3.2d, v17.2d
ld1 {v20.d}[1], [x0], x1
zip1 v29.2d, v4.2d, v18.2d
zip2 v30.2d, v4.2d, v18.2d
ld1 {v21.d}[0], [x0], x1
uaddw v23.8h, v23.8h, v19.8b
uaddw2 v24.8h, v24.8h, v19.16b
ld1 {v21.d}[1], [x0], x1
sqxtun v23.8b, v23.8h
sqxtun2 v23.16b, v24.8h
ld1 {v22.d}[0], [x0], x1
uaddw v24.8h, v25.8h, v20.8b
uaddw2 v25.8h, v26.8h, v20.16b
ld1 {v22.d}[1], [x0], x1
sqxtun v24.8b, v24.8h
sqxtun2 v24.16b, v25.8h
st1 {v23.d}[0], [x9], x1
uaddw v25.8h, v27.8h, v21.8b
uaddw2 v26.8h, v28.8h, v21.16b
st1 {v23.d}[1], [x9], x1
sqxtun v25.8b, v25.8h
sqxtun2 v25.16b, v26.8h
st1 {v24.d}[0], [x9], x1
uaddw v26.8h, v29.8h, v22.8b
uaddw2 v27.8h, v30.8h, v22.16b
st1 {v24.d}[1], [x9], x1
sqxtun v26.8b, v26.8h
sqxtun2 v26.16b, v27.8h
st1 {v25.d}[0], [x9], x1
st1 {v25.d}[1], [x9], x1
st1 {v26.d}[0], [x9], x1
st1 {v26.d}[1], [x9], x1
idct_end
endfunc
function ff_simple_idct_neon, export=1
idct_start x0
mov x2, x0
idct_row4_neon v24, v25, v26, v27, 1
idct_row4_neon v28, v29, v30, v31, 2
sub x2, x2, #128
bl idct_col4_neon1
sshr v1.8h, v7.8h, #COL_SHIFT-16
sshr v2.8h, v16.8h, #COL_SHIFT-16
sshr v3.8h, v17.8h, #COL_SHIFT-16
sshr v4.8h, v18.8h, #COL_SHIFT-16
bl idct_col4_neon2
sshr v7.8h, v7.8h, #COL_SHIFT-16
sshr v16.8h, v16.8h, #COL_SHIFT-16
sshr v17.8h, v17.8h, #COL_SHIFT-16
sshr v18.8h, v18.8h, #COL_SHIFT-16
zip1 v23.2d, v1.2d, v7.2d
zip2 v24.2d, v1.2d, v7.2d
st1 {v23.2d,v24.2d}, [x2], #32
zip1 v25.2d, v2.2d, v16.2d
zip2 v26.2d, v2.2d, v16.2d
st1 {v25.2d,v26.2d}, [x2], #32
zip1 v27.2d, v3.2d, v17.2d
zip2 v28.2d, v3.2d, v17.2d
st1 {v27.2d,v28.2d}, [x2], #32
zip1 v29.2d, v4.2d, v18.2d
zip2 v30.2d, v4.2d, v18.2d
st1 {v29.2d,v30.2d}, [x2], #32
idct_end
endfunc
@@ -0,0 +1,40 @@
/*
* Copyright (c) 2010 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/aarch64/cpu.h"
#include "libavutil/attributes.h"
#include "libavutil/internal.h"
#include "libavcodec/synth_filter.h"
void ff_synth_filter_float_neon(AVTXContext *imdct,
float *synth_buf_ptr, int *synth_buf_offset,
float synth_buf2[32], const float window[512],
float out[32], float in[32],
float scale, av_tx_fn imdct_fn);
av_cold void ff_synth_filter_init_aarch64(SynthFilterContext *s)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags))
s->synth_filter_float = ff_synth_filter_float_neon;
}
@@ -0,0 +1,119 @@
/*
* Copyright (c) 2010 Mans Rullgard <mans@mansr.com>
* Copyright (c) 2015 Janne Grunau <janne-libav@jannau.net>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
.macro inner_loop
ld1 {v29.4s}, [x9], x15
ld1 {v28.4s}, [x8], x15
ld1 {v30.4s}, [x10], x15
ld1 {v31.4s}, [x11], x15
rev64 v28.4s, v28.4s
ld1 {v24.4s}, [x4], x15
ld1 {v25.4s}, [x5], x15
rev64 v31.4s, v31.4s
ld1 {v26.4s}, [x6], x15
fmla v5.4s, v25.4s, v29.4s
ld1 {v27.4s}, [x7], x15
ext v28.16b, v28.16b, v28.16b, #8
ext v31.16b, v31.16b, v31.16b, #8
fmla v6.4s, v26.4s, v30.4s
fmls v4.4s, v24.4s, v28.4s
fmla v7.4s, v27.4s, v31.4s
.endm
function ff_synth_filter_float_neon, export=1
mov x9, x7 // imdct_fn parameter
ldr w7, [x2] // *synth_buf_offset
sxtw x7, w7
stp x3, x4, [sp, #-64]!
add x1, x1, x7, lsl #2 // synth_buf
sub w8, w7, #32
stp x5, x1, [sp, #16]
and x7, x7, #~63
and w8, w8, #511
AARCH64_SIGN_LINK_REGISTER
stp x7, x30, [sp, #32]
str w8, [x2]
str s0, [sp, #48]
mov x2, x6 // in
mov x3, #4 // sizeof(float)
blr x9
ldp x13, x9, [sp, #16] // out, synth_buf
ldp x0, x30, [sp, #32] // *synth_buf_offset
AARCH64_VALIDATE_LINK_REGISTER
ldr s0, [sp, #48]
ldp x2, x4, [sp], #64 // synct_buf_2, window
add x3, x2, #16*4 // synct_buf_2 + 16
add x14, x13, #16*4 // out + 16
add x8, x9, #12*4
mov x15, #64*4
mov x1, #4
1:
add x10, x9, #16*4 // synth_buf
add x11, x8, #16*4
add x5, x4, #16*4 // window
add x6, x4, #32*4
add x7, x4, #48*4
ld1 {v4.4s}, [x2] // a
ld1 {v5.4s}, [x3] // b
movi v6.4s, #0 // c
movi v7.4s, #0 // d
mov x12, #512
2:
sub x12, x12, #64
cmp x12, x0
inner_loop
b.gt 2b
sub x8, x8, #512*4
sub x9, x9, #512*4
cbz x12, 4f
sub x10, x10, #512*4
sub x11, x11, #512*4
3:
subs x12, x12, #64
inner_loop
b.gt 3b
4:
subs x1, x1, #1
fmul v4.4s, v4.4s, v0.s[0]
fmul v5.4s, v5.4s, v0.s[0]
st1 {v6.4s}, [x2], #16
st1 {v7.4s}, [x3], #16
st1 {v4.4s}, [x13], #16
st1 {v5.4s}, [x14], #16
b.le 10f
sub x4, x4, #508*4 // window
add x9, x9, #4*4 // synth_buf
sub x8, x8, #4*4 // synth_buf
b 1b
10:
ret
endfunc
@@ -0,0 +1,141 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavutil/intreadwrite.h"
#include "libavcodec/vc1dsp.h"
#include "config.h"
void ff_vc1_inv_trans_8x8_neon(int16_t *block);
void ff_vc1_inv_trans_8x4_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_inv_trans_4x8_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_inv_trans_4x4_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_inv_trans_8x8_dc_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_inv_trans_8x4_dc_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_inv_trans_4x8_dc_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_inv_trans_4x4_dc_neon(uint8_t *dest, ptrdiff_t stride, int16_t *block);
void ff_vc1_v_loop_filter4_neon(uint8_t *src, ptrdiff_t stride, int pq);
void ff_vc1_h_loop_filter4_neon(uint8_t *src, ptrdiff_t stride, int pq);
void ff_vc1_v_loop_filter8_neon(uint8_t *src, ptrdiff_t stride, int pq);
void ff_vc1_h_loop_filter8_neon(uint8_t *src, ptrdiff_t stride, int pq);
void ff_vc1_v_loop_filter16_neon(uint8_t *src, ptrdiff_t stride, int pq);
void ff_vc1_h_loop_filter16_neon(uint8_t *src, ptrdiff_t stride, int pq);
void ff_put_vc1_chroma_mc8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_vc1_chroma_mc8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_put_vc1_chroma_mc4_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
void ff_avg_vc1_chroma_mc4_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride,
int h, int x, int y);
int ff_vc1_unescape_buffer_helper_neon(const uint8_t *src, int size, uint8_t *dst);
static int vc1_unescape_buffer_neon(const uint8_t *src, int size, uint8_t *dst)
{
/* Dealing with starting and stopping, and removing escape bytes, are
* comparatively less time-sensitive, so are more clearly expressed using
* a C wrapper around the assembly inner loop. Note that we assume a
* little-endian machine that supports unaligned loads. */
int dsize = 0;
while (size >= 4)
{
int found = 0;
while (!found && (((uintptr_t) dst) & 7) && size >= 4)
{
found = (AV_RL32(src) &~ 0x03000000) == 0x00030000;
if (!found)
{
*dst++ = *src++;
--size;
++dsize;
}
}
if (!found)
{
int skip = size - ff_vc1_unescape_buffer_helper_neon(src, size, dst);
dst += skip;
src += skip;
size -= skip;
dsize += skip;
while (!found && size >= 4)
{
found = (AV_RL32(src) &~ 0x03000000) == 0x00030000;
if (!found)
{
*dst++ = *src++;
--size;
++dsize;
}
}
}
if (found)
{
*dst++ = *src++;
*dst++ = *src++;
++src;
size -= 3;
dsize += 2;
}
}
while (size > 0)
{
*dst++ = *src++;
--size;
++dsize;
}
return dsize;
}
av_cold void ff_vc1dsp_init_aarch64(VC1DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
dsp->vc1_inv_trans_8x8 = ff_vc1_inv_trans_8x8_neon;
dsp->vc1_inv_trans_8x4 = ff_vc1_inv_trans_8x4_neon;
dsp->vc1_inv_trans_4x8 = ff_vc1_inv_trans_4x8_neon;
dsp->vc1_inv_trans_4x4 = ff_vc1_inv_trans_4x4_neon;
dsp->vc1_inv_trans_8x8_dc = ff_vc1_inv_trans_8x8_dc_neon;
dsp->vc1_inv_trans_8x4_dc = ff_vc1_inv_trans_8x4_dc_neon;
dsp->vc1_inv_trans_4x8_dc = ff_vc1_inv_trans_4x8_dc_neon;
dsp->vc1_inv_trans_4x4_dc = ff_vc1_inv_trans_4x4_dc_neon;
dsp->vc1_v_loop_filter4 = ff_vc1_v_loop_filter4_neon;
dsp->vc1_h_loop_filter4 = ff_vc1_h_loop_filter4_neon;
dsp->vc1_v_loop_filter8 = ff_vc1_v_loop_filter8_neon;
dsp->vc1_h_loop_filter8 = ff_vc1_h_loop_filter8_neon;
dsp->vc1_v_loop_filter16 = ff_vc1_v_loop_filter16_neon;
dsp->vc1_h_loop_filter16 = ff_vc1_h_loop_filter16_neon;
dsp->put_no_rnd_vc1_chroma_pixels_tab[0] = ff_put_vc1_chroma_mc8_neon;
dsp->avg_no_rnd_vc1_chroma_pixels_tab[0] = ff_avg_vc1_chroma_mc8_neon;
dsp->put_no_rnd_vc1_chroma_pixels_tab[1] = ff_put_vc1_chroma_mc4_neon;
dsp->avg_no_rnd_vc1_chroma_pixels_tab[1] = ff_avg_vc1_chroma_mc4_neon;
dsp->vc1_unescape_buffer = vc1_unescape_buffer_neon;
}
}
File diff suppressed because it is too large. Load diff
+29
View File
@@ -0,0 +1,29 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
function ff_prefetch_aarch64, export=1
1:
subs w2, w2, #2
prfm pldl1strm, [x0]
prfm pldl1strm, [x0, x1]
add x0, x0, x1, lsl #1
b.gt 1b
ret
endfunc
+32
View File
@@ -0,0 +1,32 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/videodsp.h"
void ff_prefetch_aarch64(const uint8_t *mem, ptrdiff_t stride, int h);
av_cold void ff_videodsp_init_aarch64(VideoDSPContext *ctx, int bpc)
{
int cpu_flags = av_get_cpu_flags();
if (have_armv8(cpu_flags))
ctx->prefetch = ff_prefetch_aarch64;
}
@@ -0,0 +1,34 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/vorbisdsp.h"
void ff_vorbis_inverse_coupling_neon(float *mag, float *ang,
ptrdiff_t blocksize);
av_cold void ff_vorbisdsp_init_aarch64(VorbisDSPContext *c)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
c->vorbis_inverse_coupling = ff_vorbis_inverse_coupling_neon;
}
}
@@ -0,0 +1,82 @@
/*
* Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
function ff_vorbis_inverse_coupling_neon, export=1
movi v20.4s, #1<<7, lsl #24
subs x2, x2, #4
mov x3, x0
mov x4, x1
b.eq 3f
ld1 {v7.4s}, [x1], #16
ld1 {v6.4s}, [x0], #16
cmle v4.4s, v7.4s, #0
and v5.16b, v6.16b, v20.16b
eor v7.16b, v7.16b, v5.16b
and v2.16b, v7.16b, v4.16b
bic v3.16b, v7.16b, v4.16b
fadd v7.4s, v6.4s, v2.4s
fsub v6.4s, v6.4s, v3.4s
1: ld1 {v1.4s}, [x1], #16
ld1 {v0.4s}, [x0], #16
cmle v4.4s, v1.4s, #0
and v5.16b, v0.16b, v20.16b
eor v1.16b, v1.16b, v5.16b
st1 {v7.4s}, [x3], #16
st1 {v6.4s}, [x4], #16
and v2.16b, v1.16b, v4.16b
bic v3.16b, v1.16b, v4.16b
fadd v1.4s, v0.4s, v2.4s
fsub v0.4s, v0.4s, v3.4s
subs x2, x2, #8
b.le 2f
ld1 {v7.4s}, [x1], #16
ld1 {v6.4s}, [x0], #16
cmle v4.4s, v7.4s, #0
and v5.16b, v6.16b, v20.16b
eor v7.16b, v7.16b, v5.16b
st1 {v1.4s}, [x3], #16
st1 {v0.4s}, [x4], #16
and v2.16b, v7.16b, v4.16b
bic v3.16b, v7.16b, v4.16b
fadd v7.4s, v6.4s, v2.4s
fsub v6.4s, v6.4s, v3.4s
b 1b
2: st1 {v1.4s}, [x3], #16
st1 {v0.4s}, [x4], #16
b.lt ret
3: ld1 {v1.4s}, [x1]
ld1 {v0.4s}, [x0]
cmle v4.4s, v1.4s, #0
and v5.16b, v0.16b, v20.16b
eor v1.16b, v1.16b, v5.16b
and v2.16b, v1.16b, v4.16b
bic v3.16b, v1.16b, v4.16b
fadd v1.4s, v0.4s, v2.4s
fsub v0.4s, v0.4s, v3.4s
st1 {v1.4s}, [x0], #16
st1 {v0.4s}, [x1], #16
ret:
ret
endfunc
+75
View File
@@ -0,0 +1,75 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_AARCH64_VP8DSP_H
#define AVCODEC_AARCH64_VP8DSP_H
#include "libavcodec/vp8dsp.h"
#define VP8_LF_Y(hv, inner, opt) \
void ff_vp8_##hv##_loop_filter16##inner##_##opt(uint8_t *dst, \
ptrdiff_t stride, \
int flim_E, int flim_I, \
int hev_thresh)
#define VP8_LF_UV(hv, inner, opt) \
void ff_vp8_##hv##_loop_filter8uv##inner##_##opt(uint8_t *dstU, \
uint8_t *dstV, \
ptrdiff_t stride, \
int flim_E, int flim_I, \
int hev_thresh)
#define VP8_LF_SIMPLE(hv, opt) \
void ff_vp8_##hv##_loop_filter16_simple_##opt(uint8_t *dst, \
ptrdiff_t stride, \
int flim)
#define VP8_LF_HV(inner, opt) \
VP8_LF_Y(h, inner, opt); \
VP8_LF_Y(v, inner, opt); \
VP8_LF_UV(h, inner, opt); \
VP8_LF_UV(v, inner, opt)
#define VP8_LF(opt) \
VP8_LF_HV(, opt); \
VP8_LF_HV(_inner, opt); \
VP8_LF_SIMPLE(h, opt); \
VP8_LF_SIMPLE(v, opt)
#define VP8_MC(n, opt) \
void ff_put_vp8_##n##_##opt(uint8_t *dst, ptrdiff_t dststride, \
const uint8_t *src, ptrdiff_t srcstride,\
int h, int x, int y)
#define VP8_EPEL(w, opt) \
VP8_MC(pixels ## w, opt); \
VP8_MC(epel ## w ## _h4, opt); \
VP8_MC(epel ## w ## _h6, opt); \
VP8_MC(epel ## w ## _v4, opt); \
VP8_MC(epel ## w ## _h4v4, opt); \
VP8_MC(epel ## w ## _h6v4, opt); \
VP8_MC(epel ## w ## _v6, opt); \
VP8_MC(epel ## w ## _h4v6, opt); \
VP8_MC(epel ## w ## _h6v6, opt)
#define VP8_BILIN(w, opt) \
VP8_MC(bilin ## w ## _h, opt); \
VP8_MC(bilin ## w ## _v, opt); \
VP8_MC(bilin ## w ## _hv, opt)
#endif /* AVCODEC_AARCH64_VP8DSP_H */
@@ -0,0 +1,124 @@
/*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/vp8dsp.h"
#include "vp8dsp.h"
void ff_vp8_luma_dc_wht_neon(int16_t block[4][4][16], int16_t dc[16]);
void ff_vp8_idct_add_neon(uint8_t *dst, int16_t block[16], ptrdiff_t stride);
void ff_vp8_idct_dc_add_neon(uint8_t *dst, int16_t block[16], ptrdiff_t stride);
void ff_vp8_idct_dc_add4y_neon(uint8_t *dst, int16_t block[4][16], ptrdiff_t stride);
void ff_vp8_idct_dc_add4uv_neon(uint8_t *dst, int16_t block[4][16], ptrdiff_t stride);
VP8_LF(neon);
VP8_EPEL(16, neon);
VP8_EPEL(8, neon);
VP8_EPEL(4, neon);
VP8_BILIN(16, neon);
VP8_BILIN(8, neon);
VP8_BILIN(4, neon);
av_cold void ff_vp78dsp_init_aarch64(VP8DSPContext *dsp)
{
if (!have_neon(av_get_cpu_flags()))
return;
dsp->put_vp8_epel_pixels_tab[0][0][0] = ff_put_vp8_pixels16_neon;
dsp->put_vp8_epel_pixels_tab[0][0][2] = ff_put_vp8_epel16_h6_neon;
dsp->put_vp8_epel_pixels_tab[0][2][0] = ff_put_vp8_epel16_v6_neon;
dsp->put_vp8_epel_pixels_tab[0][2][2] = ff_put_vp8_epel16_h6v6_neon;
dsp->put_vp8_epel_pixels_tab[1][0][0] = ff_put_vp8_pixels8_neon;
dsp->put_vp8_epel_pixels_tab[1][0][1] = ff_put_vp8_epel8_h4_neon;
dsp->put_vp8_epel_pixels_tab[1][0][2] = ff_put_vp8_epel8_h6_neon;
dsp->put_vp8_epel_pixels_tab[1][1][0] = ff_put_vp8_epel8_v4_neon;
dsp->put_vp8_epel_pixels_tab[1][1][1] = ff_put_vp8_epel8_h4v4_neon;
dsp->put_vp8_epel_pixels_tab[1][1][2] = ff_put_vp8_epel8_h6v4_neon;
dsp->put_vp8_epel_pixels_tab[1][2][0] = ff_put_vp8_epel8_v6_neon;
dsp->put_vp8_epel_pixels_tab[1][2][1] = ff_put_vp8_epel8_h4v6_neon;
dsp->put_vp8_epel_pixels_tab[1][2][2] = ff_put_vp8_epel8_h6v6_neon;
dsp->put_vp8_epel_pixels_tab[2][0][1] = ff_put_vp8_epel4_h4_neon;
dsp->put_vp8_epel_pixels_tab[2][0][2] = ff_put_vp8_epel4_h6_neon;
dsp->put_vp8_epel_pixels_tab[2][1][0] = ff_put_vp8_epel4_v4_neon;
dsp->put_vp8_epel_pixels_tab[2][1][1] = ff_put_vp8_epel4_h4v4_neon;
dsp->put_vp8_epel_pixels_tab[2][1][2] = ff_put_vp8_epel4_h6v4_neon;
dsp->put_vp8_epel_pixels_tab[2][2][0] = ff_put_vp8_epel4_v6_neon;
dsp->put_vp8_epel_pixels_tab[2][2][1] = ff_put_vp8_epel4_h4v6_neon;
dsp->put_vp8_epel_pixels_tab[2][2][2] = ff_put_vp8_epel4_h6v6_neon;
dsp->put_vp8_bilinear_pixels_tab[0][0][0] = ff_put_vp8_pixels16_neon;
dsp->put_vp8_bilinear_pixels_tab[0][0][1] = ff_put_vp8_bilin16_h_neon;
dsp->put_vp8_bilinear_pixels_tab[0][0][2] = ff_put_vp8_bilin16_h_neon;
dsp->put_vp8_bilinear_pixels_tab[0][1][0] = ff_put_vp8_bilin16_v_neon;
dsp->put_vp8_bilinear_pixels_tab[0][1][1] = ff_put_vp8_bilin16_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[0][1][2] = ff_put_vp8_bilin16_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[0][2][0] = ff_put_vp8_bilin16_v_neon;
dsp->put_vp8_bilinear_pixels_tab[0][2][1] = ff_put_vp8_bilin16_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[0][2][2] = ff_put_vp8_bilin16_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[1][0][0] = ff_put_vp8_pixels8_neon;
dsp->put_vp8_bilinear_pixels_tab[1][0][1] = ff_put_vp8_bilin8_h_neon;
dsp->put_vp8_bilinear_pixels_tab[1][0][2] = ff_put_vp8_bilin8_h_neon;
dsp->put_vp8_bilinear_pixels_tab[1][1][0] = ff_put_vp8_bilin8_v_neon;
dsp->put_vp8_bilinear_pixels_tab[1][1][1] = ff_put_vp8_bilin8_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[1][1][2] = ff_put_vp8_bilin8_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[1][2][0] = ff_put_vp8_bilin8_v_neon;
dsp->put_vp8_bilinear_pixels_tab[1][2][1] = ff_put_vp8_bilin8_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[1][2][2] = ff_put_vp8_bilin8_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[2][0][1] = ff_put_vp8_bilin4_h_neon;
dsp->put_vp8_bilinear_pixels_tab[2][0][2] = ff_put_vp8_bilin4_h_neon;
dsp->put_vp8_bilinear_pixels_tab[2][1][0] = ff_put_vp8_bilin4_v_neon;
dsp->put_vp8_bilinear_pixels_tab[2][1][1] = ff_put_vp8_bilin4_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[2][1][2] = ff_put_vp8_bilin4_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[2][2][0] = ff_put_vp8_bilin4_v_neon;
dsp->put_vp8_bilinear_pixels_tab[2][2][1] = ff_put_vp8_bilin4_hv_neon;
dsp->put_vp8_bilinear_pixels_tab[2][2][2] = ff_put_vp8_bilin4_hv_neon;
}
av_cold void ff_vp8dsp_init_aarch64(VP8DSPContext *dsp)
{
if (!have_neon(av_get_cpu_flags()))
return;
dsp->vp8_luma_dc_wht = ff_vp8_luma_dc_wht_neon;
dsp->vp8_idct_add = ff_vp8_idct_add_neon;
dsp->vp8_idct_dc_add = ff_vp8_idct_dc_add_neon;
dsp->vp8_idct_dc_add4y = ff_vp8_idct_dc_add4y_neon;
dsp->vp8_idct_dc_add4uv = ff_vp8_idct_dc_add4uv_neon;
dsp->vp8_v_loop_filter16y = ff_vp8_v_loop_filter16_neon;
dsp->vp8_h_loop_filter16y = ff_vp8_h_loop_filter16_neon;
dsp->vp8_v_loop_filter8uv = ff_vp8_v_loop_filter8uv_neon;
dsp->vp8_h_loop_filter8uv = ff_vp8_h_loop_filter8uv_neon;
dsp->vp8_v_loop_filter16y_inner = ff_vp8_v_loop_filter16_inner_neon;
dsp->vp8_h_loop_filter16y_inner = ff_vp8_h_loop_filter16_inner_neon;
dsp->vp8_v_loop_filter8uv_inner = ff_vp8_v_loop_filter8uv_inner_neon;
dsp->vp8_h_loop_filter8uv_inner = ff_vp8_h_loop_filter8uv_inner_neon;
dsp->vp8_v_loop_filter_simple = ff_vp8_v_loop_filter16_simple_neon;
dsp->vp8_h_loop_filter_simple = ff_vp8_h_loop_filter16_simple_neon;
}
File diff suppressed because it is too large. Load diff
+29
View File
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2017 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef AVCODEC_AARCH64_VP9DSP_INIT_H
#define AVCODEC_AARCH64_VP9DSP_INIT_H
#include "libavcodec/vp9dsp.h"
void ff_vp9dsp_init_10bpp_aarch64(VP9DSPContext *dsp);
void ff_vp9dsp_init_12bpp_aarch64(VP9DSPContext *dsp);
#endif /* AVCODEC_AARCH64_VP9DSP_INIT_H */
@@ -0,0 +1,23 @@
/*
* Copyright (c) 2017 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#define BPP 10
#define INIT_FUNC ff_vp9dsp_init_10bpp_aarch64
#include "vp9dsp_init_16bpp_aarch64_template.c"
@@ -0,0 +1,23 @@
/*
* Copyright (c) 2017 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#define BPP 12
#define INIT_FUNC ff_vp9dsp_init_12bpp_aarch64
#include "vp9dsp_init_16bpp_aarch64_template.c"
@@ -0,0 +1,274 @@
/*
* Copyright (c) 2017 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/internal.h"
#include "libavutil/mem_internal.h"
#include "libavutil/aarch64/cpu.h"
#include "vp9dsp_init.h"
#define declare_fpel(type, sz, suffix) \
void ff_vp9_##type##sz##suffix##_neon(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, \
int h, int mx, int my)
#define decl_mc_func(op, filter, dir, sz, bpp) \
void ff_vp9_##op##_##filter##sz##_##dir##_##bpp##_neon(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, \
int h, int mx, int my)
#define define_8tap_2d_fn(op, filter, sz, bpp) \
static void op##_##filter##sz##_hv_##bpp##_neon(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, \
ptrdiff_t src_stride, \
int h, int mx, int my) \
{ \
LOCAL_ALIGNED_16(uint8_t, temp, [((1 + (sz < 64)) * sz + 8) * sz * 2]); \
/* We only need h + 7 lines, but the horizontal filter assumes an \
* even number of rows, so filter h + 8 lines here. */ \
ff_vp9_put_##filter##sz##_h_##bpp##_neon(temp, 2 * sz, \
src - 3 * src_stride, src_stride, \
h + 8, mx, 0); \
ff_vp9_##op##_##filter##sz##_v_##bpp##_neon(dst, dst_stride, \
temp + 3 * 2 * sz, 2 * sz, \
h, 0, my); \
}
#define decl_filter_funcs(op, dir, sz, bpp) \
decl_mc_func(op, regular, dir, sz, bpp); \
decl_mc_func(op, sharp, dir, sz, bpp); \
decl_mc_func(op, smooth, dir, sz, bpp)
#define decl_mc_funcs(sz, bpp) \
decl_filter_funcs(put, h, sz, bpp); \
decl_filter_funcs(avg, h, sz, bpp); \
decl_filter_funcs(put, v, sz, bpp); \
decl_filter_funcs(avg, v, sz, bpp); \
decl_filter_funcs(put, hv, sz, bpp); \
decl_filter_funcs(avg, hv, sz, bpp)
#define ff_vp9_copy32_neon ff_vp9_copy32_aarch64
#define ff_vp9_copy64_neon ff_vp9_copy64_aarch64
#define ff_vp9_copy128_neon ff_vp9_copy128_aarch64
declare_fpel(copy, 128, );
declare_fpel(copy, 64, );
declare_fpel(copy, 32, );
declare_fpel(copy, 16, );
declare_fpel(copy, 8, );
declare_fpel(avg, 64, _16);
declare_fpel(avg, 32, _16);
declare_fpel(avg, 16, _16);
declare_fpel(avg, 8, _16);
declare_fpel(avg, 4, _16);
decl_mc_funcs(64, BPP);
decl_mc_funcs(32, BPP);
decl_mc_funcs(16, BPP);
decl_mc_funcs(8, BPP);
decl_mc_funcs(4, BPP);
#define define_8tap_2d_funcs(sz, bpp) \
define_8tap_2d_fn(put, regular, sz, bpp) \
define_8tap_2d_fn(put, sharp, sz, bpp) \
define_8tap_2d_fn(put, smooth, sz, bpp) \
define_8tap_2d_fn(avg, regular, sz, bpp) \
define_8tap_2d_fn(avg, sharp, sz, bpp) \
define_8tap_2d_fn(avg, smooth, sz, bpp)
define_8tap_2d_funcs(64, BPP)
define_8tap_2d_funcs(32, BPP)
define_8tap_2d_funcs(16, BPP)
define_8tap_2d_funcs(8, BPP)
define_8tap_2d_funcs(4, BPP)
static av_cold void vp9dsp_mc_init_aarch64(VP9DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
#define init_fpel(idx1, idx2, sz, type, suffix) \
dsp->mc[idx1][FILTER_8TAP_SMOOTH ][idx2][0][0] = \
dsp->mc[idx1][FILTER_8TAP_REGULAR][idx2][0][0] = \
dsp->mc[idx1][FILTER_8TAP_SHARP ][idx2][0][0] = \
dsp->mc[idx1][FILTER_BILINEAR ][idx2][0][0] = ff_vp9_##type##sz##suffix
#define init_copy(idx, sz, suffix) \
init_fpel(idx, 0, sz, copy, suffix)
#define init_avg(idx, sz, suffix) \
init_fpel(idx, 1, sz, avg, suffix)
#define init_copy_avg(idx, sz1, sz2) \
init_copy(idx, sz2, _neon); \
init_avg (idx, sz1, _16_neon)
if (have_armv8(cpu_flags)) {
init_copy(0, 128, _aarch64);
init_copy(1, 64, _aarch64);
init_copy(2, 32, _aarch64);
}
if (have_neon(cpu_flags)) {
#define init_mc_func(idx1, idx2, op, filter, fname, dir, mx, my, sz, pfx, bpp) \
dsp->mc[idx1][filter][idx2][mx][my] = pfx##op##_##fname##sz##_##dir##_##bpp##_neon
#define init_mc_funcs(idx, dir, mx, my, sz, pfx, bpp) \
init_mc_func(idx, 0, put, FILTER_8TAP_REGULAR, regular, dir, mx, my, sz, pfx, bpp); \
init_mc_func(idx, 0, put, FILTER_8TAP_SHARP, sharp, dir, mx, my, sz, pfx, bpp); \
init_mc_func(idx, 0, put, FILTER_8TAP_SMOOTH, smooth, dir, mx, my, sz, pfx, bpp); \
init_mc_func(idx, 1, avg, FILTER_8TAP_REGULAR, regular, dir, mx, my, sz, pfx, bpp); \
init_mc_func(idx, 1, avg, FILTER_8TAP_SHARP, sharp, dir, mx, my, sz, pfx, bpp); \
init_mc_func(idx, 1, avg, FILTER_8TAP_SMOOTH, smooth, dir, mx, my, sz, pfx, bpp)
#define init_mc_funcs_dirs(idx, sz, bpp) \
init_mc_funcs(idx, v, 0, 1, sz, ff_vp9_, bpp); \
init_mc_funcs(idx, h, 1, 0, sz, ff_vp9_, bpp); \
init_mc_funcs(idx, hv, 1, 1, sz, , bpp)
init_avg(0, 64, _16_neon);
init_avg(1, 32, _16_neon);
init_avg(2, 16, _16_neon);
init_copy_avg(3, 8, 16);
init_copy_avg(4, 4, 8);
init_mc_funcs_dirs(0, 64, BPP);
init_mc_funcs_dirs(1, 32, BPP);
init_mc_funcs_dirs(2, 16, BPP);
init_mc_funcs_dirs(3, 8, BPP);
init_mc_funcs_dirs(4, 4, BPP);
}
}
#define define_itxfm2(type_a, type_b, sz, bpp) \
void ff_vp9_##type_a##_##type_b##_##sz##x##sz##_add_##bpp##_neon(uint8_t *_dst, \
ptrdiff_t stride, \
int16_t *_block, int eob)
#define define_itxfm(type_a, type_b, sz, bpp) define_itxfm2(type_a, type_b, sz, bpp)
#define define_itxfm_funcs(sz, bpp) \
define_itxfm(idct, idct, sz, bpp); \
define_itxfm(iadst, idct, sz, bpp); \
define_itxfm(idct, iadst, sz, bpp); \
define_itxfm(iadst, iadst, sz, bpp)
define_itxfm_funcs(4, BPP);
define_itxfm_funcs(8, BPP);
define_itxfm_funcs(16, BPP);
define_itxfm(idct, idct, 32, BPP);
define_itxfm(iwht, iwht, 4, BPP);
static av_cold void vp9dsp_itxfm_init_aarch64(VP9DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
#define init_itxfm2(tx, sz, bpp) \
dsp->itxfm_add[tx][DCT_DCT] = ff_vp9_idct_idct_##sz##_add_##bpp##_neon; \
dsp->itxfm_add[tx][DCT_ADST] = ff_vp9_iadst_idct_##sz##_add_##bpp##_neon; \
dsp->itxfm_add[tx][ADST_DCT] = ff_vp9_idct_iadst_##sz##_add_##bpp##_neon; \
dsp->itxfm_add[tx][ADST_ADST] = ff_vp9_iadst_iadst_##sz##_add_##bpp##_neon
#define init_itxfm(tx, sz, bpp) init_itxfm2(tx, sz, bpp)
#define init_idct2(tx, nm, bpp) \
dsp->itxfm_add[tx][DCT_DCT] = \
dsp->itxfm_add[tx][ADST_DCT] = \
dsp->itxfm_add[tx][DCT_ADST] = \
dsp->itxfm_add[tx][ADST_ADST] = ff_vp9_##nm##_add_##bpp##_neon
#define init_idct(tx, nm, bpp) init_idct2(tx, nm, bpp)
init_itxfm(TX_4X4, 4x4, BPP);
init_itxfm(TX_8X8, 8x8, BPP);
init_itxfm(TX_16X16, 16x16, BPP);
init_idct(TX_32X32, idct_idct_32x32, BPP);
init_idct(4, iwht_iwht_4x4, BPP);
}
}
#define define_loop_filter(dir, wd, size, bpp) \
void ff_vp9_loop_filter_##dir##_##wd##_##size##_##bpp##_neon(uint8_t *dst, ptrdiff_t stride, int E, int I, int H)
#define define_loop_filters(wd, size, bpp) \
define_loop_filter(h, wd, size, bpp); \
define_loop_filter(v, wd, size, bpp)
define_loop_filters(4, 8, BPP);
define_loop_filters(8, 8, BPP);
define_loop_filters(16, 8, BPP);
define_loop_filters(16, 16, BPP);
define_loop_filters(44, 16, BPP);
define_loop_filters(48, 16, BPP);
define_loop_filters(84, 16, BPP);
define_loop_filters(88, 16, BPP);
static av_cold void vp9dsp_loopfilter_init_aarch64(VP9DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
#define init_lpf_func_8(idx1, idx2, dir, wd, bpp) \
dsp->loop_filter_8[idx1][idx2] = ff_vp9_loop_filter_##dir##_##wd##_8_##bpp##_neon
#define init_lpf_func_16(idx, dir, bpp) \
dsp->loop_filter_16[idx] = ff_vp9_loop_filter_##dir##_16_16_##bpp##_neon
#define init_lpf_func_mix2(idx1, idx2, idx3, dir, wd, bpp) \
dsp->loop_filter_mix2[idx1][idx2][idx3] = ff_vp9_loop_filter_##dir##_##wd##_16_##bpp##_neon
#define init_lpf_funcs_8_wd(idx, wd, bpp) \
init_lpf_func_8(idx, 0, h, wd, bpp); \
init_lpf_func_8(idx, 1, v, wd, bpp)
#define init_lpf_funcs_16(bpp) \
init_lpf_func_16(0, h, bpp); \
init_lpf_func_16(1, v, bpp)
#define init_lpf_funcs_mix2_wd(idx1, idx2, wd, bpp) \
init_lpf_func_mix2(idx1, idx2, 0, h, wd, bpp); \
init_lpf_func_mix2(idx1, idx2, 1, v, wd, bpp)
#define init_lpf_funcs_8(bpp) \
init_lpf_funcs_8_wd(0, 4, bpp); \
init_lpf_funcs_8_wd(1, 8, bpp); \
init_lpf_funcs_8_wd(2, 16, bpp)
#define init_lpf_funcs_mix2(bpp) \
init_lpf_funcs_mix2_wd(0, 0, 44, bpp); \
init_lpf_funcs_mix2_wd(0, 1, 48, bpp); \
init_lpf_funcs_mix2_wd(1, 0, 84, bpp); \
init_lpf_funcs_mix2_wd(1, 1, 88, bpp)
init_lpf_funcs_8(BPP);
init_lpf_funcs_16(BPP);
init_lpf_funcs_mix2(BPP);
}
}
av_cold void INIT_FUNC(VP9DSPContext *dsp)
{
vp9dsp_mc_init_aarch64(dsp);
vp9dsp_loopfilter_init_aarch64(dsp);
vp9dsp_itxfm_init_aarch64(dsp);
}
@@ -0,0 +1,259 @@
/*
* Copyright (c) 2016 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <stdint.h>
#include "libavutil/attributes.h"
#include "libavutil/internal.h"
#include "libavutil/mem_internal.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/vp9dsp.h"
#include "vp9dsp_init.h"
#define declare_fpel(type, sz) \
void ff_vp9_##type##sz##_neon(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, \
int h, int mx, int my)
#define declare_copy_avg(sz) \
declare_fpel(copy, sz); \
declare_fpel(avg , sz)
#define decl_mc_func(op, filter, dir, sz) \
void ff_vp9_##op##_##filter##sz##_##dir##_neon(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, \
int h, int mx, int my)
#define define_8tap_2d_fn(op, filter, sz) \
static void op##_##filter##sz##_hv_neon(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, \
int h, int mx, int my) \
{ \
LOCAL_ALIGNED_16(uint8_t, temp, [((1 + (sz < 64)) * sz + 8) * sz]); \
/* We only need h + 7 lines, but the horizontal filter assumes an \
* even number of rows, so filter h + 8 lines here. */ \
ff_vp9_put_##filter##sz##_h_neon(temp, sz, \
src - 3 * src_stride, src_stride, \
h + 8, mx, 0); \
ff_vp9_##op##_##filter##sz##_v_neon(dst, dst_stride, \
temp + 3 * sz, sz, \
h, 0, my); \
}
#define decl_filter_funcs(op, dir, sz) \
decl_mc_func(op, regular, dir, sz); \
decl_mc_func(op, sharp, dir, sz); \
decl_mc_func(op, smooth, dir, sz)
#define decl_mc_funcs(sz) \
decl_filter_funcs(put, h, sz); \
decl_filter_funcs(avg, h, sz); \
decl_filter_funcs(put, v, sz); \
decl_filter_funcs(avg, v, sz); \
decl_filter_funcs(put, hv, sz); \
decl_filter_funcs(avg, hv, sz)
#define ff_vp9_copy32_neon ff_vp9_copy32_aarch64
#define ff_vp9_copy64_neon ff_vp9_copy64_aarch64
declare_copy_avg(64);
declare_copy_avg(32);
declare_copy_avg(16);
declare_copy_avg(8);
declare_copy_avg(4);
decl_mc_funcs(64);
decl_mc_funcs(32);
decl_mc_funcs(16);
decl_mc_funcs(8);
decl_mc_funcs(4);
#define define_8tap_2d_funcs(sz) \
define_8tap_2d_fn(put, regular, sz) \
define_8tap_2d_fn(put, sharp, sz) \
define_8tap_2d_fn(put, smooth, sz) \
define_8tap_2d_fn(avg, regular, sz) \
define_8tap_2d_fn(avg, sharp, sz) \
define_8tap_2d_fn(avg, smooth, sz)
define_8tap_2d_funcs(64)
define_8tap_2d_funcs(32)
define_8tap_2d_funcs(16)
define_8tap_2d_funcs(8)
define_8tap_2d_funcs(4)
static av_cold void vp9dsp_mc_init_aarch64(VP9DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
#define init_fpel(idx1, idx2, sz, type, suffix) \
dsp->mc[idx1][FILTER_8TAP_SMOOTH ][idx2][0][0] = \
dsp->mc[idx1][FILTER_8TAP_REGULAR][idx2][0][0] = \
dsp->mc[idx1][FILTER_8TAP_SHARP ][idx2][0][0] = \
dsp->mc[idx1][FILTER_BILINEAR ][idx2][0][0] = ff_vp9_##type##sz##suffix
#define init_copy(idx, sz, suffix) \
init_fpel(idx, 0, sz, copy, suffix)
#define init_avg(idx, sz, suffix) \
init_fpel(idx, 1, sz, avg, suffix)
#define init_copy_avg(idx, sz) \
init_copy(idx, sz, _neon); \
init_avg (idx, sz, _neon)
if (have_armv8(cpu_flags)) {
init_copy(0, 64, _aarch64);
init_copy(1, 32, _aarch64);
}
if (have_neon(cpu_flags)) {
#define init_mc_func(idx1, idx2, op, filter, fname, dir, mx, my, sz, pfx) \
dsp->mc[idx1][filter][idx2][mx][my] = pfx##op##_##fname##sz##_##dir##_neon
#define init_mc_funcs(idx, dir, mx, my, sz, pfx) \
init_mc_func(idx, 0, put, FILTER_8TAP_REGULAR, regular, dir, mx, my, sz, pfx); \
init_mc_func(idx, 0, put, FILTER_8TAP_SHARP, sharp, dir, mx, my, sz, pfx); \
init_mc_func(idx, 0, put, FILTER_8TAP_SMOOTH, smooth, dir, mx, my, sz, pfx); \
init_mc_func(idx, 1, avg, FILTER_8TAP_REGULAR, regular, dir, mx, my, sz, pfx); \
init_mc_func(idx, 1, avg, FILTER_8TAP_SHARP, sharp, dir, mx, my, sz, pfx); \
init_mc_func(idx, 1, avg, FILTER_8TAP_SMOOTH, smooth, dir, mx, my, sz, pfx)
#define init_mc_funcs_dirs(idx, sz) \
init_mc_funcs(idx, h, 1, 0, sz, ff_vp9_); \
init_mc_funcs(idx, v, 0, 1, sz, ff_vp9_); \
init_mc_funcs(idx, hv, 1, 1, sz,)
init_avg(0, 64, _neon);
init_avg(1, 32, _neon);
init_copy_avg(2, 16);
init_copy_avg(3, 8);
init_copy_avg(4, 4);
init_mc_funcs_dirs(0, 64);
init_mc_funcs_dirs(1, 32);
init_mc_funcs_dirs(2, 16);
init_mc_funcs_dirs(3, 8);
init_mc_funcs_dirs(4, 4);
}
}
#define define_itxfm(type_a, type_b, sz) \
void ff_vp9_##type_a##_##type_b##_##sz##x##sz##_add_neon(uint8_t *_dst, \
ptrdiff_t stride, \
int16_t *_block, int eob)
#define define_itxfm_funcs(sz) \
define_itxfm(idct, idct, sz); \
define_itxfm(iadst, idct, sz); \
define_itxfm(idct, iadst, sz); \
define_itxfm(iadst, iadst, sz)
define_itxfm_funcs(4);
define_itxfm_funcs(8);
define_itxfm_funcs(16);
define_itxfm(idct, idct, 32);
define_itxfm(iwht, iwht, 4);
static av_cold void vp9dsp_itxfm_init_aarch64(VP9DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
#define init_itxfm(tx, sz) \
dsp->itxfm_add[tx][DCT_DCT] = ff_vp9_idct_idct_##sz##_add_neon; \
dsp->itxfm_add[tx][DCT_ADST] = ff_vp9_iadst_idct_##sz##_add_neon; \
dsp->itxfm_add[tx][ADST_DCT] = ff_vp9_idct_iadst_##sz##_add_neon; \
dsp->itxfm_add[tx][ADST_ADST] = ff_vp9_iadst_iadst_##sz##_add_neon
#define init_idct(tx, nm) \
dsp->itxfm_add[tx][DCT_DCT] = \
dsp->itxfm_add[tx][ADST_DCT] = \
dsp->itxfm_add[tx][DCT_ADST] = \
dsp->itxfm_add[tx][ADST_ADST] = ff_vp9_##nm##_add_neon
init_itxfm(TX_4X4, 4x4);
init_itxfm(TX_8X8, 8x8);
init_itxfm(TX_16X16, 16x16);
init_idct(TX_32X32, idct_idct_32x32);
init_idct(4, iwht_iwht_4x4);
}
}
#define define_loop_filter(dir, wd, len) \
void ff_vp9_loop_filter_##dir##_##wd##_##len##_neon(uint8_t *dst, ptrdiff_t stride, int E, int I, int H)
#define define_loop_filters(wd, len) \
define_loop_filter(h, wd, len); \
define_loop_filter(v, wd, len)
define_loop_filters(4, 8);
define_loop_filters(8, 8);
define_loop_filters(16, 8);
define_loop_filters(16, 16);
define_loop_filters(44, 16);
define_loop_filters(48, 16);
define_loop_filters(84, 16);
define_loop_filters(88, 16);
static av_cold void vp9dsp_loopfilter_init_aarch64(VP9DSPContext *dsp)
{
int cpu_flags = av_get_cpu_flags();
if (have_neon(cpu_flags)) {
dsp->loop_filter_8[0][1] = ff_vp9_loop_filter_v_4_8_neon;
dsp->loop_filter_8[0][0] = ff_vp9_loop_filter_h_4_8_neon;
dsp->loop_filter_8[1][1] = ff_vp9_loop_filter_v_8_8_neon;
dsp->loop_filter_8[1][0] = ff_vp9_loop_filter_h_8_8_neon;
dsp->loop_filter_8[2][1] = ff_vp9_loop_filter_v_16_8_neon;
dsp->loop_filter_8[2][0] = ff_vp9_loop_filter_h_16_8_neon;
dsp->loop_filter_16[0] = ff_vp9_loop_filter_h_16_16_neon;
dsp->loop_filter_16[1] = ff_vp9_loop_filter_v_16_16_neon;
dsp->loop_filter_mix2[0][0][0] = ff_vp9_loop_filter_h_44_16_neon;
dsp->loop_filter_mix2[0][0][1] = ff_vp9_loop_filter_v_44_16_neon;
dsp->loop_filter_mix2[0][1][0] = ff_vp9_loop_filter_h_48_16_neon;
dsp->loop_filter_mix2[0][1][1] = ff_vp9_loop_filter_v_48_16_neon;
dsp->loop_filter_mix2[1][0][0] = ff_vp9_loop_filter_h_84_16_neon;
dsp->loop_filter_mix2[1][0][1] = ff_vp9_loop_filter_v_84_16_neon;
dsp->loop_filter_mix2[1][1][0] = ff_vp9_loop_filter_h_88_16_neon;
dsp->loop_filter_mix2[1][1][1] = ff_vp9_loop_filter_v_88_16_neon;
}
}
av_cold void ff_vp9dsp_init_aarch64(VP9DSPContext *dsp, int bpp)
{
if (bpp == 10) {
ff_vp9dsp_init_10bpp_aarch64(dsp);
return;
} else if (bpp == 12) {
ff_vp9dsp_init_12bpp_aarch64(dsp);
return;
} else if (bpp != 8)
return;
vp9dsp_mc_init_aarch64(dsp);
vp9dsp_loopfilter_init_aarch64(dsp);
vp9dsp_itxfm_init_aarch64(dsp);
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,861 @@
/*
* Copyright (c) 2017 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
// The input to and output from this macro is in the registers v16-v31,
// and v0-v7 are used as scratch registers.
// p7 = v16 .. p3 = v20, p0 = v23, q0 = v24, q3 = v27, q7 = v31
// Depending on the width of the loop filter, we either use v16-v19
// and v28-v31 as temp registers, or v8-v15.
.macro loop_filter wd, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8
dup v0.8h, w2 // E
dup v2.8h, w3 // I
dup v3.8h, w4 // H
uabd v4.8h, v20.8h, v21.8h // abs(p3 - p2)
uabd v5.8h, v21.8h, v22.8h // abs(p2 - p1)
uabd v6.8h, v22.8h, v23.8h // abs(p1 - p0)
uabd v7.8h, v24.8h, v25.8h // abs(q0 - q1)
uabd \tmp1\().8h, v25.8h, v26.8h // abs(q1 - q2)
uabd \tmp2\().8h, v26.8h, v27.8h // abs(q2 - q3)
umax v4.8h, v4.8h, v5.8h
umax v5.8h, v6.8h, v7.8h
umax \tmp1\().8h, \tmp1\().8h, \tmp2\().8h
uabd v6.8h, v23.8h, v24.8h // abs(p0 - q0)
umax v4.8h, v4.8h, v5.8h
add v6.8h, v6.8h, v6.8h // abs(p0 - q0) * 2
uabd v5.8h, v22.8h, v25.8h // abs(p1 - q1)
umax v4.8h, v4.8h, \tmp1\().8h // max(abs(p3 - p2), ..., abs(q2 - q3))
ushr v5.8h, v5.8h, #1
cmhs v4.8h, v2.8h, v4.8h // max(abs()) <= I
add v6.8h, v6.8h, v5.8h // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1
cmhs v6.8h, v0.8h, v6.8h
and v4.16b, v4.16b, v6.16b // fm
// If no pixels need filtering, just exit as soon as possible
mov x11, v4.d[0]
mov x12, v4.d[1]
adds x11, x11, x12
b.ne 1f
ret x10
1:
.if \wd >= 8
dup v0.8h, w5
uabd v6.8h, v20.8h, v23.8h // abs(p3 - p0)
uabd v2.8h, v21.8h, v23.8h // abs(p2 - p0)
uabd v1.8h, v22.8h, v23.8h // abs(p1 - p0)
uabd \tmp1\().8h, v25.8h, v24.8h // abs(q1 - q0)
uabd \tmp2\().8h, v26.8h, v24.8h // abs(q2 - q0)
uabd \tmp3\().8h, v27.8h, v24.8h // abs(q3 - q0)
umax v6.8h, v6.8h, v2.8h
umax v1.8h, v1.8h, \tmp1\().8h
umax \tmp2\().8h, \tmp2\().8h, \tmp3\().8h
.if \wd == 16
uabd v7.8h, v16.8h, v23.8h // abs(p7 - p0)
umax v6.8h, v6.8h, v1.8h
uabd v2.8h, v17.8h, v23.8h // abs(p6 - p0)
umax v6.8h, v6.8h, \tmp2\().8h
uabd v1.8h, v18.8h, v23.8h // abs(p5 - p0)
cmhs v6.8h, v0.8h, v6.8h // flat8in
uabd v8.8h, v19.8h, v23.8h // abs(p4 - p0)
and v6.16b, v6.16b, v4.16b // flat8in && fm
uabd v9.8h, v28.8h, v24.8h // abs(q4 - q0)
bic v4.16b, v4.16b, v6.16b // fm && !flat8in
uabd v10.8h, v29.8h, v24.8h // abs(q5 - q0)
uabd v11.8h, v30.8h, v24.8h // abs(q6 - q0)
uabd v12.8h, v31.8h, v24.8h // abs(q7 - q0)
umax v7.8h, v7.8h, v2.8h
umax v1.8h, v1.8h, v8.8h
umax v9.8h, v9.8h, v10.8h
umax v11.8h, v11.8h, v12.8h
// The rest of the calculation of flat8out is interleaved below
.else
// The rest of the calculation of flat8in is interleaved below
.endif
.endif
// Calculate the normal inner loop filter for 2 or 4 pixels
uabd v5.8h, v22.8h, v23.8h // abs(p1 - p0)
.if \wd == 16
umax v7.8h, v7.8h, v1.8h
umax v9.8h, v9.8h, v11.8h
.elseif \wd == 8
umax v6.8h, v6.8h, v1.8h
.endif
uabd v1.8h, v25.8h, v24.8h // abs(q1 - q0)
.if \wd == 16
umax v7.8h, v7.8h, v9.8h
.elseif \wd == 8
umax v6.8h, v6.8h, \tmp2\().8h
.endif
dup \tmp2\().8h, w6 // left shift for saturation
sub \tmp1\().8h, v22.8h, v25.8h // p1 - q1
neg \tmp6\().8h, \tmp2\().8h // negative left shift after saturation
umax v5.8h, v5.8h, v1.8h // max(abs(p1 - p0), abs(q1 - q0))
sub \tmp3\().8h, v24.8h, v23.8h // q0 - p0
movi \tmp5\().8h, #3
.if \wd == 8
cmhs v6.8h, v0.8h, v6.8h // flat8in
.endif
cmhs v5.8h, v3.8h, v5.8h // !hev
.if \wd == 8
and v6.16b, v6.16b, v4.16b // flat8in && fm
.endif
sqshl \tmp1\().8h, \tmp1\().8h, \tmp2\().8h
.if \wd == 16
cmhs v7.8h, v0.8h, v7.8h // flat8out
.elseif \wd == 8
bic v4.16b, v4.16b, v6.16b // fm && !flat8in
.endif
and v5.16b, v5.16b, v4.16b // !hev && fm && !flat8in
.if \wd == 16
and v7.16b, v7.16b, v6.16b // flat8out && flat8in && fm
.endif
sshl \tmp1\().8h, \tmp1\().8h, \tmp6\().8h // av_clip_int2p(p1 - q1, BIT_DEPTH - 1)
mul \tmp3\().8h, \tmp3\().8h, \tmp5\().8h // 3 * (q0 - p0)
bic \tmp1\().16b, \tmp1\().16b, v5.16b // if (!hev) av_clip_int8 = 0
movi v2.8h, #4
add \tmp3\().8h, \tmp3\().8h, \tmp1\().8h // 3 * (q0 - p0) [+ av_clip_int8(p1 - q1)]
movi v3.8h, #3
sqshl \tmp1\().8h, \tmp3\().8h, \tmp2\().8h
movi \tmp5\().8h, #0
sshl \tmp1\().8h, \tmp1\().8h, \tmp6\().8h // av_clip_int2p(3 * (q0 - p0) [+ av_clip_int2p(p1 - q1)], BIT_DEPTH - 1) = f
dup \tmp6\().8h, w7 // max pixel value
.if \wd == 16
bic v6.16b, v6.16b, v7.16b // fm && flat8in && !flat8out
.endif
ushr \tmp2\().8h, \tmp6\().8h, #1 // (1 << (BIT_DEPTH - 1)) - 1
add \tmp3\().8h, \tmp1\().8h, v2.8h // f + 4
add \tmp4\().8h, \tmp1\().8h, v3.8h // f + 3
smin \tmp3\().8h, \tmp3\().8h, \tmp2\().8h // FFMIN(f + 4, (1 << (BIT_DEPTH - 1)) - 1)
smin \tmp4\().8h, \tmp4\().8h, \tmp2\().8h // FFMIN(f + 3, (1 << (BIT_DEPTH - 1)) - 1)
sshr \tmp3\().8h, \tmp3\().8h, #3 // f1
sshr \tmp4\().8h, \tmp4\().8h, #3 // f2
add v0.8h, v23.8h, \tmp4\().8h // p0 + f2
sub v2.8h, v24.8h, \tmp3\().8h // q0 - f1
smin v0.8h, v0.8h, \tmp6\().8h
smin v2.8h, v2.8h, \tmp6\().8h
srshr \tmp3\().8h, \tmp3\().8h, #1 // f = (f1 + 1) >> 1
smax v0.8h, v0.8h, \tmp5\().8h // out p0
smax v2.8h, v2.8h, \tmp5\().8h // out q0
bit v23.16b, v0.16b, v4.16b // if (fm && !flat8in)
bit v24.16b, v2.16b, v4.16b
add v0.8h, v22.8h, \tmp3\().8h // p1 + f
sub v2.8h, v25.8h, \tmp3\().8h // q1 - f
.if \wd >= 8
mov x11, v6.d[0]
.endif
smin v0.8h, v0.8h, \tmp6\().8h
smin v2.8h, v2.8h, \tmp6\().8h
.if \wd >= 8
mov x12, v6.d[1]
.endif
smax v0.8h, v0.8h, \tmp5\().8h // out p1
smax v2.8h, v2.8h, \tmp5\().8h // out q1
.if \wd >= 8
adds x11, x11, x12
.endif
bit v22.16b, v0.16b, v5.16b // if (!hev && fm && !flat8in)
bit v25.16b, v2.16b, v5.16b
// If no pixels need flat8in, jump to flat8out
// (or to a writeout of the inner 4 pixels, for wd=8)
.if \wd >= 8
.if \wd == 16
b.eq 6f
.else
b.ne 1f
ret x13
1:
.endif
// flat8in
add \tmp1\().8h, v20.8h, v21.8h
add \tmp3\().8h, v22.8h, v25.8h
add \tmp5\().8h, v20.8h, v22.8h
add \tmp7\().8h, v23.8h, v26.8h
add v0.8h, \tmp1\().8h, \tmp1\().8h
add v0.8h, v0.8h, v23.8h
add v0.8h, v0.8h, v24.8h
add v0.8h, v0.8h, \tmp5\().8h
sub \tmp3\().8h, \tmp3\().8h, \tmp1\().8h
sub \tmp7\().8h, \tmp7\().8h, \tmp5\().8h
urshr v2.8h, v0.8h, #3 // out p2
add v0.8h, v0.8h, \tmp3\().8h
add \tmp1\().8h, v20.8h, v23.8h
add \tmp3\().8h, v24.8h, v27.8h
urshr v3.8h, v0.8h, #3 // out p1
add v0.8h, v0.8h, \tmp7\().8h
sub \tmp3\().8h, \tmp3\().8h, \tmp1\().8h
add \tmp5\().8h, v21.8h, v24.8h
add \tmp7\().8h, v25.8h, v27.8h
urshr v4.8h, v0.8h, #3 // out p0
add v0.8h, v0.8h, \tmp3\().8h
sub \tmp7\().8h, \tmp7\().8h, \tmp5\().8h
add \tmp1\().8h, v22.8h, v25.8h
add \tmp3\().8h, v26.8h, v27.8h
urshr v5.8h, v0.8h, #3 // out q0
add v0.8h, v0.8h, \tmp7\().8h
sub \tmp3\().8h, \tmp3\().8h, \tmp1\().8h
urshr \tmp5\().8h, v0.8h, #3 // out q1
add v0.8h, v0.8h, \tmp3\().8h
// The output here is written back into the input registers. This doesn't
// matter for the flat8part below, since we only update those pixels
// which won't be touched below.
bit v21.16b, v2.16b, v6.16b
bit v22.16b, v3.16b, v6.16b
bit v23.16b, v4.16b, v6.16b
urshr \tmp6\().8h, v0.8h, #3 // out q2
bit v24.16b, v5.16b, v6.16b
bit v25.16b, \tmp5\().16b, v6.16b
bit v26.16b, \tmp6\().16b, v6.16b
.endif
.if \wd == 16
6:
orr v2.16b, v6.16b, v7.16b
mov x11, v2.d[0]
mov x12, v2.d[1]
adds x11, x11, x12
b.ne 1f
// If no pixels needed flat8in nor flat8out, jump to a
// writeout of the inner 4 pixels
ret x14
1:
mov x11, v7.d[0]
mov x12, v7.d[1]
adds x11, x11, x12
b.ne 1f
// If no pixels need flat8out, jump to a writeout of the inner 6 pixels
ret x15
1:
// flat8out
// This writes all outputs into v2-v17 (skipping v6 and v16).
// If this part is skipped, the output is read from v21-v26 (which is the input
// to this section).
shl v0.8h, v16.8h, #3 // 8 * v16
sub v0.8h, v0.8h, v16.8h // 7 * v16
add v0.8h, v0.8h, v17.8h
add v8.8h, v17.8h, v18.8h
add v10.8h, v19.8h, v20.8h
add v0.8h, v0.8h, v8.8h
add v8.8h, v16.8h, v17.8h
add v12.8h, v21.8h, v22.8h
add v0.8h, v0.8h, v10.8h
add v10.8h, v18.8h, v25.8h
add v14.8h, v23.8h, v24.8h
sub v10.8h, v10.8h, v8.8h
add v0.8h, v0.8h, v12.8h
add v0.8h, v0.8h, v14.8h
add v12.8h, v16.8h, v18.8h
add v14.8h, v19.8h, v26.8h
urshr v2.8h, v0.8h, #4
add v0.8h, v0.8h, v10.8h
add v8.8h, v16.8h, v19.8h
add v10.8h, v20.8h, v27.8h
sub v14.8h, v14.8h, v12.8h
bif v2.16b, v17.16b, v7.16b
urshr v3.8h , v0.8h, #4
add v0.8h, v0.8h, v14.8h
add v12.8h, v16.8h, v20.8h
add v14.8h, v21.8h, v28.8h
sub v10.8h, v10.8h, v8.8h
bif v3.16b, v18.16b, v7.16b
urshr v4.8h, v0.8h, #4
add v0.8h, v0.8h, v10.8h
add v8.8h, v16.8h, v21.8h
add v10.8h, v22.8h, v29.8h
sub v14.8h, v14.8h, v12.8h
bif v4.16b, v19.16b, v7.16b
urshr v5.8h, v0.8h, #4
add v0.8h, v0.8h, v14.8h
add v12.8h, v16.8h, v22.8h
add v14.8h, v23.8h, v30.8h
sub v10.8h, v10.8h, v8.8h
bif v5.16b, v20.16b, v7.16b
urshr v6.8h, v0.8h, #4
add v0.8h, v0.8h, v10.8h
add v10.8h, v16.8h, v23.8h
sub v14.8h, v14.8h, v12.8h
add v12.8h, v24.8h, v31.8h
bif v6.16b, v21.16b, v7.16b
urshr v8.8h, v0.8h, #4
add v0.8h, v0.8h, v14.8h
sub v10.8h, v12.8h, v10.8h
add v12.8h, v17.8h, v24.8h
add v14.8h, v25.8h, v31.8h
bif v8.16b, v22.16b, v7.16b
urshr v9.8h, v0.8h, #4
add v0.8h, v0.8h, v10.8h
sub v14.8h, v14.8h, v12.8h
add v12.8h, v26.8h, v31.8h
bif v9.16b, v23.16b, v7.16b
urshr v10.8h, v0.8h, #4
add v0.8h, v0.8h, v14.8h
add v14.8h, v18.8h, v25.8h
add v18.8h, v19.8h, v26.8h
sub v12.8h, v12.8h, v14.8h
add v14.8h, v27.8h, v31.8h
bif v10.16b, v24.16b, v7.16b
urshr v11.8h, v0.8h, #4
add v0.8h, v0.8h, v12.8h
add v12.8h, v20.8h, v27.8h
sub v14.8h, v14.8h, v18.8h
add v18.8h, v28.8h, v31.8h
bif v11.16b, v25.16b, v7.16b
sub v18.8h, v18.8h, v12.8h
urshr v12.8h, v0.8h, #4
add v0.8h, v0.8h, v14.8h
add v14.8h, v21.8h, v28.8h
add v20.8h, v29.8h, v31.8h
bif v12.16b, v26.16b, v7.16b
urshr v13.8h, v0.8h, #4
add v0.8h, v0.8h, v18.8h
sub v20.8h, v20.8h, v14.8h
add v18.8h, v22.8h, v29.8h
add v22.8h, v30.8h, v31.8h
bif v13.16b, v27.16b, v7.16b
urshr v14.8h, v0.8h, #4
add v0.8h, v0.8h, v20.8h
sub v22.8h, v22.8h, v18.8h
bif v14.16b, v28.16b, v7.16b
urshr v15.8h, v0.8h, #4
add v0.8h, v0.8h, v22.8h
bif v15.16b, v29.16b, v7.16b
urshr v17.8h, v0.8h, #4
bif v17.16b, v30.16b, v7.16b
.endif
.endm
// For wd <= 8, we use v16-v19 and v28-v31 for temp registers,
// while we need those for inputs/outputs in wd=16 and use v8-v15
// for temp registers there instead.
function vp9_loop_filter_4
loop_filter 4, v16, v17, v18, v19, v28, v29, v30, v31
ret
endfunc
function vp9_loop_filter_8
loop_filter 8, v16, v17, v18, v19, v28, v29, v30, v31
ret
endfunc
function vp9_loop_filter_16
loop_filter 16, v8, v9, v10, v11, v12, v13, v14, v15
ret
endfunc
.macro loop_filter_4
bl vp9_loop_filter_4
.endm
.macro loop_filter_8
// calculate alternative 'return' targets
adr x13, 6f
bl vp9_loop_filter_8
.endm
.macro loop_filter_16
// calculate alternative 'return' targets
adr x14, 7f
adr x15, 8f
bl vp9_loop_filter_16
.endm
// The public functions in this file have got the following signature:
// void loop_filter(uint8_t *dst, ptrdiff_t stride, int mb_lim, int lim, int hev_thr);
.macro bpp_frontend func, bpp, push
function ff_\func\()_\bpp\()_neon, export=1
.if \push
mov x16, x30
stp d8, d9, [sp, #-0x40]!
stp d14, d15, [sp, #0x30]
stp d12, d13, [sp, #0x20]
stp d10, d11, [sp, #0x10]
.endif
lsl w2, w2, #\bpp - 8
lsl w3, w3, #\bpp - 8
lsl w4, w4, #\bpp - 8
mov x5, #1 << (\bpp - 8)
mov x6, #16 - \bpp
mov x7, #((1 << \bpp) - 1)
.if \push
bl \func\()_16_neon
ldp d10, d11, [sp, #0x10]
ldp d12, d13, [sp, #0x20]
ldp d14, d15, [sp, #0x30]
ldp d8, d9, [sp], #0x40
ret x16
.else
b \func\()_16_neon
.endif
endfunc
.endm
.macro bpp_frontends func, push=0
bpp_frontend \func, 10, \push
bpp_frontend \func, 12, \push
.endm
.macro bpp_frontend_rep func, suffix, int_suffix, dir, bpp, push
function ff_\func\()_\suffix\()_\bpp\()_neon, export=1
mov x16, x30
.if \push
stp d8, d9, [sp, #-0x40]!
stp d14, d15, [sp, #0x30]
stp d12, d13, [sp, #0x20]
stp d10, d11, [sp, #0x10]
.endif
lsl w2, w2, #\bpp - 8
lsl w3, w3, #\bpp - 8
lsl w4, w4, #\bpp - 8
mov x5, #1 << (\bpp - 8)
mov x6, #16 - \bpp
mov x7, #((1 << \bpp) - 1)
bl \func\()_\int_suffix\()_16_neon
.ifc \dir,h
add x0, x0, x1, lsl #3
.else
add x0, x0, #16
.endif
bl \func\()_\int_suffix\()_16_neon
.if \push
ldp d10, d11, [sp, #0x10]
ldp d12, d13, [sp, #0x20]
ldp d14, d15, [sp, #0x30]
ldp d8, d9, [sp], 0x40
.endif
ret x16
endfunc
.endm
.macro bpp_frontends_rep func, suffix, int_suffix, dir, push=0
bpp_frontend_rep \func, \suffix, \int_suffix, \dir, 10, \push
bpp_frontend_rep \func, \suffix, \int_suffix, \dir, 12, \push
.endm
.macro bpp_frontend_mix2 wd1, wd2, dir, bpp
function ff_vp9_loop_filter_\dir\()_\wd1\()\wd2\()_16_\bpp\()_neon, export=1
mov x16, x30
lsr w8, w2, #8
lsr w14, w3, #8
lsr w15, w4, #8
and w2, w2, #0xff
and w3, w3, #0xff
and w4, w4, #0xff
lsl w2, w2, #\bpp - 8
lsl w3, w3, #\bpp - 8
lsl w4, w4, #\bpp - 8
mov x5, #1 << (\bpp - 8)
mov x6, #16 - \bpp
mov x7, #((1 << \bpp) - 1)
bl vp9_loop_filter_\dir\()_\wd1\()_8_16_neon
.ifc \dir,h
add x0, x0, x1, lsl #3
.else
add x0, x0, #16
.endif
lsl w2, w8, #\bpp - 8
lsl w3, w14, #\bpp - 8
lsl w4, w15, #\bpp - 8
bl vp9_loop_filter_\dir\()_\wd2\()_8_16_neon
ret x16
endfunc
.endm
.macro bpp_frontends_mix2 wd1, wd2
bpp_frontend_mix2 \wd1, \wd2, v, 10
bpp_frontend_mix2 \wd1, \wd2, v, 12
bpp_frontend_mix2 \wd1, \wd2, h, 10
bpp_frontend_mix2 \wd1, \wd2, h, 12
.endm
function vp9_loop_filter_v_4_8_16_neon
mov x10, x30
sub x9, x0, x1, lsl #2
ld1 {v20.8h}, [x9], x1 // p3
ld1 {v24.8h}, [x0], x1 // q0
ld1 {v21.8h}, [x9], x1 // p2
ld1 {v25.8h}, [x0], x1 // q1
ld1 {v22.8h}, [x9], x1 // p1
ld1 {v26.8h}, [x0], x1 // q2
ld1 {v23.8h}, [x9], x1 // p0
ld1 {v27.8h}, [x0], x1 // q3
sub x0, x0, x1, lsl #2
sub x9, x9, x1, lsl #1
loop_filter_4
st1 {v22.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
sub x0, x0, x1, lsl #1
ret x10
endfunc
bpp_frontends vp9_loop_filter_v_4_8
function vp9_loop_filter_h_4_8_16_neon
mov x10, x30
sub x9, x0, #8
add x0, x9, x1, lsl #2
ld1 {v20.8h}, [x9], x1
ld1 {v24.8h}, [x0], x1
ld1 {v21.8h}, [x9], x1
ld1 {v25.8h}, [x0], x1
ld1 {v22.8h}, [x9], x1
ld1 {v26.8h}, [x0], x1
ld1 {v23.8h}, [x9], x1
ld1 {v27.8h}, [x0], x1
sub x9, x9, x1, lsl #2
sub x0, x0, x1, lsl #3
add x0, x0, #8
transpose_8x8H v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
loop_filter_4
// Move x9 forward by 2 pixels; we don't need to rewrite the
// outermost 2 pixels since they aren't changed.
add x9, x9, #4
add x0, x9, x1, lsl #2
// We only will write the mid 4 pixels back; after the loop filter,
// these are in v22, v23, v24, v25, ordered as rows (8x4 pixels).
// We need to transpose them to columns, done with a 4x8 transpose
// (which in practice is two 4x4 transposes of the two 4x4 halves
// of the 8x4 pixels; into 4x8 pixels).
transpose_4x8H v22, v23, v24, v25, v26, v27, v28, v29
st1 {v22.d}[0], [x9], x1
st1 {v22.d}[1], [x0], x1
st1 {v23.d}[0], [x9], x1
st1 {v23.d}[1], [x0], x1
st1 {v24.d}[0], [x9], x1
st1 {v24.d}[1], [x0], x1
st1 {v25.d}[0], [x9], x1
st1 {v25.d}[1], [x0], x1
sub x0, x0, x1, lsl #3
add x0, x0, #4
ret x10
endfunc
bpp_frontends vp9_loop_filter_h_4_8
function vp9_loop_filter_v_8_8_16_neon
mov x10, x30
sub x9, x0, x1, lsl #2
ld1 {v20.8h}, [x9], x1 // p3
ld1 {v24.8h}, [x0], x1 // q0
ld1 {v21.8h}, [x9], x1 // p2
ld1 {v25.8h}, [x0], x1 // q1
ld1 {v22.8h}, [x9], x1 // p1
ld1 {v26.8h}, [x0], x1 // q2
ld1 {v23.8h}, [x9], x1 // p0
ld1 {v27.8h}, [x0], x1 // q3
sub x9, x9, x1, lsl #2
sub x0, x0, x1, lsl #2
add x9, x9, x1
loop_filter_8
st1 {v21.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v22.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v26.8h}, [x0], x1
sub x0, x0, x1, lsl #1
sub x0, x0, x1
ret x10
6:
sub x9, x0, x1, lsl #1
st1 {v22.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
sub x0, x0, x1, lsl #1
ret x10
endfunc
bpp_frontends vp9_loop_filter_v_8_8
function vp9_loop_filter_h_8_8_16_neon
mov x10, x30
sub x9, x0, #8
add x0, x9, x1, lsl #2
ld1 {v20.8h}, [x9], x1
ld1 {v24.8h}, [x0], x1
ld1 {v21.8h}, [x9], x1
ld1 {v25.8h}, [x0], x1
ld1 {v22.8h}, [x9], x1
ld1 {v26.8h}, [x0], x1
ld1 {v23.8h}, [x9], x1
ld1 {v27.8h}, [x0], x1
sub x9, x9, x1, lsl #2
sub x0, x0, x1, lsl #3
add x0, x0, #8
transpose_8x8H v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
loop_filter_8
add x0, x9, x1, lsl #2
// Even though only 6 pixels per row have been changed, we write the
// full 8 pixel registers.
transpose_8x8H v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
st1 {v20.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v21.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
st1 {v22.8h}, [x9], x1
st1 {v26.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v27.8h}, [x0], x1
sub x0, x0, x1, lsl #3
add x0, x0, #8
ret x10
6:
// If we didn't need to do the flat8in part, we use the same writeback
// as in loop_filter_h_4_8.
add x9, x9, #4
add x0, x9, x1, lsl #2
transpose_4x8H v22, v23, v24, v25, v26, v27, v28, v29
st1 {v22.d}[0], [x9], x1
st1 {v22.d}[1], [x0], x1
st1 {v23.d}[0], [x9], x1
st1 {v23.d}[1], [x0], x1
st1 {v24.d}[0], [x9], x1
st1 {v24.d}[1], [x0], x1
st1 {v25.d}[0], [x9], x1
st1 {v25.d}[1], [x0], x1
sub x0, x0, x1, lsl #3
add x0, x0, #4
ret x10
endfunc
bpp_frontends vp9_loop_filter_h_8_8
bpp_frontends_mix2 4, 4
bpp_frontends_mix2 4, 8
bpp_frontends_mix2 8, 4
bpp_frontends_mix2 8, 8
function vp9_loop_filter_v_16_8_16_neon
mov x10, x30
sub x9, x0, x1, lsl #3
ld1 {v16.8h}, [x9], x1 // p7
ld1 {v24.8h}, [x0], x1 // q0
ld1 {v17.8h}, [x9], x1 // p6
ld1 {v25.8h}, [x0], x1 // q1
ld1 {v18.8h}, [x9], x1 // p5
ld1 {v26.8h}, [x0], x1 // q2
ld1 {v19.8h}, [x9], x1 // p4
ld1 {v27.8h}, [x0], x1 // q3
ld1 {v20.8h}, [x9], x1 // p3
ld1 {v28.8h}, [x0], x1 // q4
ld1 {v21.8h}, [x9], x1 // p2
ld1 {v29.8h}, [x0], x1 // q5
ld1 {v22.8h}, [x9], x1 // p1
ld1 {v30.8h}, [x0], x1 // q6
ld1 {v23.8h}, [x9], x1 // p0
ld1 {v31.8h}, [x0], x1 // q7
sub x9, x9, x1, lsl #3
sub x0, x0, x1, lsl #3
add x9, x9, x1
loop_filter_16
// If we did the flat8out part, we get the output in
// v2-v17 (skipping v7 and v16). x9 points to x0 - 7 * stride,
// store v2-v9 there, and v10-v17 into x0.
st1 {v2.8h}, [x9], x1
st1 {v10.8h}, [x0], x1
st1 {v3.8h}, [x9], x1
st1 {v11.8h}, [x0], x1
st1 {v4.8h}, [x9], x1
st1 {v12.8h}, [x0], x1
st1 {v5.8h}, [x9], x1
st1 {v13.8h}, [x0], x1
st1 {v6.8h}, [x9], x1
st1 {v14.8h}, [x0], x1
st1 {v8.8h}, [x9], x1
st1 {v15.8h}, [x0], x1
st1 {v9.8h}, [x9], x1
st1 {v17.8h}, [x0], x1
sub x0, x0, x1, lsl #3
add x0, x0, x1
ret x10
8:
add x9, x9, x1, lsl #2
// If we didn't do the flat8out part, the output is left in the
// input registers.
st1 {v21.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v22.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v26.8h}, [x0], x1
sub x0, x0, x1, lsl #1
sub x0, x0, x1
ret x10
7:
sub x9, x0, x1, lsl #1
st1 {v22.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
sub x0, x0, x1, lsl #1
ret x10
endfunc
bpp_frontends vp9_loop_filter_v_16_8, push=1
bpp_frontends_rep vp9_loop_filter_v_16, 16, 8, v, push=1
function vp9_loop_filter_h_16_8_16_neon
mov x10, x30
sub x9, x0, #16
ld1 {v16.8h}, [x9], x1
ld1 {v24.8h}, [x0], x1
ld1 {v17.8h}, [x9], x1
ld1 {v25.8h}, [x0], x1
ld1 {v18.8h}, [x9], x1
ld1 {v26.8h}, [x0], x1
ld1 {v19.8h}, [x9], x1
ld1 {v27.8h}, [x0], x1
ld1 {v20.8h}, [x9], x1
ld1 {v28.8h}, [x0], x1
ld1 {v21.8h}, [x9], x1
ld1 {v29.8h}, [x0], x1
ld1 {v22.8h}, [x9], x1
ld1 {v30.8h}, [x0], x1
ld1 {v23.8h}, [x9], x1
ld1 {v31.8h}, [x0], x1
sub x0, x0, x1, lsl #3
sub x9, x9, x1, lsl #3
// The 16x8 pixels read above is in two 8x8 blocks; the left
// half in v16-v23, and the right half in v24-v31. Do two 8x8 transposes
// of this, to get one column per register.
transpose_8x8H v16, v17, v18, v19, v20, v21, v22, v23, v0, v1
transpose_8x8H v24, v25, v26, v27, v28, v29, v30, v31, v0, v1
loop_filter_16
transpose_8x8H v16, v2, v3, v4, v5, v6, v8, v9, v0, v1
transpose_8x8H v10, v11, v12, v13, v14, v15, v17, v31, v0, v1
st1 {v16.8h}, [x9], x1
st1 {v10.8h}, [x0], x1
st1 {v2.8h}, [x9], x1
st1 {v11.8h}, [x0], x1
st1 {v3.8h}, [x9], x1
st1 {v12.8h}, [x0], x1
st1 {v4.8h}, [x9], x1
st1 {v13.8h}, [x0], x1
st1 {v5.8h}, [x9], x1
st1 {v14.8h}, [x0], x1
st1 {v6.8h}, [x9], x1
st1 {v15.8h}, [x0], x1
st1 {v8.8h}, [x9], x1
st1 {v17.8h}, [x0], x1
st1 {v9.8h}, [x9], x1
st1 {v31.8h}, [x0], x1
sub x0, x0, x1, lsl #3
ret x10
8:
// The same writeback as in loop_filter_h_8_8
sub x9, x0, #8
add x0, x9, x1, lsl #2
transpose_8x8H v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
st1 {v20.8h}, [x9], x1
st1 {v24.8h}, [x0], x1
st1 {v21.8h}, [x9], x1
st1 {v25.8h}, [x0], x1
st1 {v22.8h}, [x9], x1
st1 {v26.8h}, [x0], x1
st1 {v23.8h}, [x9], x1
st1 {v27.8h}, [x0], x1
sub x0, x0, x1, lsl #3
add x0, x0, #8
ret x10
7:
// The same writeback as in loop_filter_h_4_8
sub x9, x0, #4
add x0, x9, x1, lsl #2
transpose_4x8H v22, v23, v24, v25, v26, v27, v28, v29
st1 {v22.d}[0], [x9], x1
st1 {v22.d}[1], [x0], x1
st1 {v23.d}[0], [x9], x1
st1 {v23.d}[1], [x0], x1
st1 {v24.d}[0], [x9], x1
st1 {v24.d}[1], [x0], x1
st1 {v25.d}[0], [x9], x1
st1 {v25.d}[1], [x0], x1
sub x0, x0, x1, lsl #3
add x0, x0, #4
ret x10
endfunc
bpp_frontends vp9_loop_filter_h_16_8, push=1
bpp_frontends_rep vp9_loop_filter_h_16, 16, 8, h, push=1
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,606 @@
/*
* Copyright (c) 2017 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
// All public functions in this file have the following signature:
// typedef void (*vp9_mc_func)(uint8_t *dst, ptrdiff_t dst_stride,
// const uint8_t *ref, ptrdiff_t ref_stride,
// int h, int mx, int my);
function ff_vp9_avg64_16_neon, export=1
mov x5, x0
sub x1, x1, #64
sub x3, x3, #64
1:
ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64
ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2], x3
urhadd v0.8h, v0.8h, v4.8h
urhadd v1.8h, v1.8h, v5.8h
ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x0], x1
urhadd v2.8h, v2.8h, v6.8h
urhadd v3.8h, v3.8h, v7.8h
subs w4, w4, #1
urhadd v16.8h, v16.8h, v20.8h
urhadd v17.8h, v17.8h, v21.8h
st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x5], #64
urhadd v18.8h, v18.8h, v22.8h
urhadd v19.8h, v19.8h, v23.8h
st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg32_16_neon, export=1
mov x5, x0
1:
ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x3
ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1
ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2], x3
urhadd v0.8h, v0.8h, v4.8h
urhadd v1.8h, v1.8h, v5.8h
ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x0], x1
urhadd v2.8h, v2.8h, v6.8h
urhadd v3.8h, v3.8h, v7.8h
subs w4, w4, #2
urhadd v16.8h, v16.8h, v20.8h
urhadd v17.8h, v17.8h, v21.8h
st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x5], x1
urhadd v18.8h, v18.8h, v22.8h
urhadd v19.8h, v19.8h, v23.8h
st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg16_16_neon, export=1
1:
ld1 {v2.8h, v3.8h}, [x2], x3
ld1 {v0.8h, v1.8h}, [x0]
urhadd v0.8h, v0.8h, v2.8h
urhadd v1.8h, v1.8h, v3.8h
subs w4, w4, #1
st1 {v0.8h, v1.8h}, [x0], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg8_16_neon, export=1
mov x5, x0
1:
ld1 {v2.8h}, [x2], x3
ld1 {v0.8h}, [x0], x1
ld1 {v3.8h}, [x2], x3
urhadd v0.8h, v0.8h, v2.8h
ld1 {v1.8h}, [x0], x1
urhadd v1.8h, v1.8h, v3.8h
subs w4, w4, #2
st1 {v0.8h}, [x5], x1
st1 {v1.8h}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg4_16_neon, export=1
mov x5, x0
1:
ld1 {v2.4h}, [x2], x3
ld1 {v0.4h}, [x0], x1
ld1 {v3.4h}, [x2], x3
urhadd v0.4h, v0.4h, v2.4h
ld1 {v1.4h}, [x0], x1
urhadd v1.4h, v1.4h, v3.4h
subs w4, w4, #2
st1 {v0.4h}, [x5], x1
st1 {v1.8b}, [x5], x1
b.ne 1b
ret
endfunc
// Extract a vector from src1-src2 and src4-src5 (src1-src3 and src4-src6
// for size >= 16), and multiply-accumulate into dst1 and dst5 (or
// dst1-dst2 and dst5-dst6 for size >= 8 and dst1-dst4 and dst5-dst8
// for size >= 16)
.macro extmlal dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, src1, src2, src3, src4, src5, src6, offset, size
ext v20.16b, \src1\().16b, \src2\().16b, #(2*\offset)
ext v22.16b, \src4\().16b, \src5\().16b, #(2*\offset)
smlal \dst1\().4s, v20.4h, v0.h[\offset]
smlal \dst5\().4s, v22.4h, v0.h[\offset]
.if \size >= 16
ext v21.16b, \src2\().16b, \src3\().16b, #(2*\offset)
ext v23.16b, \src5\().16b, \src6\().16b, #(2*\offset)
.endif
.if \size >= 8
smlal2 \dst2\().4s, v20.8h, v0.h[\offset]
smlal2 \dst6\().4s, v22.8h, v0.h[\offset]
.endif
.if \size >= 16
smlal \dst3\().4s, v21.4h, v0.h[\offset]
smlal \dst7\().4s, v23.4h, v0.h[\offset]
smlal2 \dst4\().4s, v21.8h, v0.h[\offset]
smlal2 \dst8\().4s, v23.8h, v0.h[\offset]
.endif
.endm
// Instantiate a horizontal filter function for the given size.
// This can work on 4, 8 or 16 pixels in parallel; for larger
// widths it will do 16 pixels at a time and loop horizontally.
// The actual width (in bytes) is passed in x5, the height in w4 and
// the filter coefficients in x9.
.macro do_8tap_h type, size
function \type\()_8tap_\size\()h
sub x2, x2, #6
add x6, x0, x1
add x7, x2, x3
add x1, x1, x1
add x3, x3, x3
// Only size >= 16 loops horizontally and needs
// reduced dst stride
.if \size >= 16
sub x1, x1, x5
.endif
// size >= 16 loads two qwords and increments r2,
// for size 4/8 it's enough with one qword and no
// postincrement
.if \size >= 16
sub x3, x3, x5
sub x3, x3, #16
.endif
// Load the filter vector
ld1 {v0.8h}, [x9]
1:
.if \size >= 16
mov x9, x5
.endif
// Load src
.if \size >= 16
ld1 {v5.8h, v6.8h, v7.8h}, [x2], #48
ld1 {v16.8h, v17.8h, v18.8h}, [x7], #48
.else
ld1 {v5.8h, v6.8h}, [x2]
ld1 {v16.8h, v17.8h}, [x7]
.endif
2:
smull v1.4s, v5.4h, v0.h[0]
smull v24.4s, v16.4h, v0.h[0]
.if \size >= 8
smull2 v2.4s, v5.8h, v0.h[0]
smull2 v25.4s, v16.8h, v0.h[0]
.endif
.if \size >= 16
smull v3.4s, v6.4h, v0.h[0]
smull v26.4s, v17.4h, v0.h[0]
smull2 v4.4s, v6.8h, v0.h[0]
smull2 v27.4s, v17.8h, v0.h[0]
.endif
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 1, \size
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 2, \size
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 3, \size
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 4, \size
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 5, \size
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 6, \size
extmlal v1, v2, v3, v4, v24, v25, v26, v27, v5, v6, v7, v16, v17, v18, 7, \size
// Round, shift and saturate
// The sqrshrun takes care of clamping negative values to zero, but
// we manually need to do umin with the max pixel value.
sqrshrun v1.4h, v1.4s, #7
sqrshrun v24.4h, v24.4s, #7
.if \size >= 8
sqrshrun2 v1.8h, v2.4s, #7
sqrshrun2 v24.8h, v25.4s, #7
umin v1.8h, v1.8h, v31.8h
umin v24.8h, v24.8h, v31.8h
.if \size >= 16
sqrshrun v2.4h, v3.4s, #7
sqrshrun v25.4h, v26.4s, #7
sqrshrun2 v2.8h, v4.4s, #7
sqrshrun2 v25.8h, v27.4s, #7
umin v2.8h, v2.8h, v31.8h
umin v25.8h, v25.8h, v31.8h
.endif
.else
umin v1.4h, v1.4h, v31.4h
umin v24.4h, v24.4h, v31.4h
.endif
// Average
.ifc \type,avg
.if \size >= 16
ld1 {v3.8h, v4.8h}, [x0]
ld1 {v29.8h, v30.8h}, [x6]
urhadd v1.8h, v1.8h, v3.8h
urhadd v2.8h, v2.8h, v4.8h
urhadd v24.8h, v24.8h, v29.8h
urhadd v25.8h, v25.8h, v30.8h
.elseif \size >= 8
ld1 {v3.8h}, [x0]
ld1 {v4.8h}, [x6]
urhadd v1.8h, v1.8h, v3.8h
urhadd v24.8h, v24.8h, v4.8h
.else
ld1 {v3.4h}, [x0]
ld1 {v4.4h}, [x6]
urhadd v1.4h, v1.4h, v3.4h
urhadd v24.4h, v24.4h, v4.4h
.endif
.endif
// Store and loop horizontally (for size >= 16)
.if \size >= 16
subs x9, x9, #32
st1 {v1.8h, v2.8h}, [x0], #32
st1 {v24.8h, v25.8h}, [x6], #32
b.eq 3f
mov v5.16b, v7.16b
mov v16.16b, v18.16b
ld1 {v6.8h, v7.8h}, [x2], #32
ld1 {v17.8h, v18.8h}, [x7], #32
b 2b
.elseif \size == 8
st1 {v1.8h}, [x0]
st1 {v24.8h}, [x6]
.else // \size == 4
st1 {v1.4h}, [x0]
st1 {v24.4h}, [x6]
.endif
3:
// Loop vertically
add x0, x0, x1
add x6, x6, x1
add x2, x2, x3
add x7, x7, x3
subs w4, w4, #2
b.ne 1b
ret
endfunc
.endm
.macro do_8tap_h_size size
do_8tap_h put, \size
do_8tap_h avg, \size
.endm
do_8tap_h_size 4
do_8tap_h_size 8
do_8tap_h_size 16
.macro do_8tap_h_func type, filter, offset, size, bpp
function ff_vp9_\type\()_\filter\()\size\()_h_\bpp\()_neon, export=1
mvni v31.8h, #((0xff << (\bpp - 8)) & 0xff), lsl #8
movrel x6, X(ff_vp9_subpel_filters), 256*\offset
cmp w5, #8
add x9, x6, w5, uxtw #4
mov x5, #2*\size
.if \size >= 16
b \type\()_8tap_16h
.else
b \type\()_8tap_\size\()h
.endif
endfunc
.endm
.macro do_8tap_h_filters size, bpp
do_8tap_h_func put, regular, 1, \size, \bpp
do_8tap_h_func avg, regular, 1, \size, \bpp
do_8tap_h_func put, sharp, 2, \size, \bpp
do_8tap_h_func avg, sharp, 2, \size, \bpp
do_8tap_h_func put, smooth, 0, \size, \bpp
do_8tap_h_func avg, smooth, 0, \size, \bpp
.endm
.macro do_8tap_h_filters_bpp bpp
do_8tap_h_filters 64, \bpp
do_8tap_h_filters 32, \bpp
do_8tap_h_filters 16, \bpp
do_8tap_h_filters 8, \bpp
do_8tap_h_filters 4, \bpp
.endm
do_8tap_h_filters_bpp 10
do_8tap_h_filters_bpp 12
// Vertical filters
// Round, shift and saturate and store reg1-reg4
.macro do_store4 reg1, reg2, reg3, reg4, tmp1, tmp2, tmp3, tmp4, minreg, type
sqrshrun \reg1\().4h, \reg1\().4s, #7
sqrshrun \reg2\().4h, \reg2\().4s, #7
sqrshrun \reg3\().4h, \reg3\().4s, #7
sqrshrun \reg4\().4h, \reg4\().4s, #7
.ifc \type,avg
ld1 {\tmp1\().4h}, [x7], x1
ld1 {\tmp2\().4h}, [x7], x1
ld1 {\tmp3\().4h}, [x7], x1
ld1 {\tmp4\().4h}, [x7], x1
.endif
umin \reg1\().4h, \reg1\().4h, \minreg\().4h
umin \reg2\().4h, \reg2\().4h, \minreg\().4h
umin \reg3\().4h, \reg3\().4h, \minreg\().4h
umin \reg4\().4h, \reg4\().4h, \minreg\().4h
.ifc \type,avg
urhadd \reg1\().4h, \reg1\().4h, \tmp1\().4h
urhadd \reg2\().4h, \reg2\().4h, \tmp2\().4h
urhadd \reg3\().4h, \reg3\().4h, \tmp3\().4h
urhadd \reg4\().4h, \reg4\().4h, \tmp4\().4h
.endif
st1 {\reg1\().4h}, [x0], x1
st1 {\reg2\().4h}, [x0], x1
st1 {\reg3\().4h}, [x0], x1
st1 {\reg4\().4h}, [x0], x1
.endm
// Round, shift and saturate and store reg1-8, where
// reg1-2, reg3-4 etc pairwise correspond to 4 rows.
.macro do_store8 reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8, minreg, type
sqrshrun \reg1\().4h, \reg1\().4s, #7
sqrshrun2 \reg1\().8h, \reg2\().4s, #7
sqrshrun \reg2\().4h, \reg3\().4s, #7
sqrshrun2 \reg2\().8h, \reg4\().4s, #7
sqrshrun \reg3\().4h, \reg5\().4s, #7
sqrshrun2 \reg3\().8h, \reg6\().4s, #7
sqrshrun \reg4\().4h, \reg7\().4s, #7
sqrshrun2 \reg4\().8h, \reg8\().4s, #7
.ifc \type,avg
ld1 {\reg5\().8h}, [x7], x1
ld1 {\reg6\().8h}, [x7], x1
ld1 {\reg7\().8h}, [x7], x1
ld1 {\reg8\().8h}, [x7], x1
.endif
umin \reg1\().8h, \reg1\().8h, \minreg\().8h
umin \reg2\().8h, \reg2\().8h, \minreg\().8h
umin \reg3\().8h, \reg3\().8h, \minreg\().8h
umin \reg4\().8h, \reg4\().8h, \minreg\().8h
.ifc \type,avg
urhadd \reg1\().8h, \reg1\().8h, \reg5\().8h
urhadd \reg2\().8h, \reg2\().8h, \reg6\().8h
urhadd \reg3\().8h, \reg3\().8h, \reg7\().8h
urhadd \reg4\().8h, \reg4\().8h, \reg8\().8h
.endif
st1 {\reg1\().8h}, [x0], x1
st1 {\reg2\().8h}, [x0], x1
st1 {\reg3\().8h}, [x0], x1
st1 {\reg4\().8h}, [x0], x1
.endm
// Evaluate the filter twice in parallel, from the inputs src1-src9 into dst1-dst2
// (src1-src8 into dst1, src2-src9 into dst2).
.macro convolve4 dst1, dst2, src1, src2, src3, src4, src5, src6, src7, src8, src9, tmp1, tmp2
smull \dst1\().4s, \src1\().4h, v0.h[0]
smull \dst2\().4s, \src2\().4h, v0.h[0]
smull \tmp1\().4s, \src2\().4h, v0.h[1]
smull \tmp2\().4s, \src3\().4h, v0.h[1]
smlal \dst1\().4s, \src3\().4h, v0.h[2]
smlal \dst2\().4s, \src4\().4h, v0.h[2]
smlal \tmp1\().4s, \src4\().4h, v0.h[3]
smlal \tmp2\().4s, \src5\().4h, v0.h[3]
smlal \dst1\().4s, \src5\().4h, v0.h[4]
smlal \dst2\().4s, \src6\().4h, v0.h[4]
smlal \tmp1\().4s, \src6\().4h, v0.h[5]
smlal \tmp2\().4s, \src7\().4h, v0.h[5]
smlal \dst1\().4s, \src7\().4h, v0.h[6]
smlal \dst2\().4s, \src8\().4h, v0.h[6]
smlal \tmp1\().4s, \src8\().4h, v0.h[7]
smlal \tmp2\().4s, \src9\().4h, v0.h[7]
add \dst1\().4s, \dst1\().4s, \tmp1\().4s
add \dst2\().4s, \dst2\().4s, \tmp2\().4s
.endm
// Evaluate the filter twice in parallel, from the inputs src1-src9 into dst1-dst4
// (src1-src8 into dst1-dst2, src2-src9 into dst3-dst4).
.macro convolve8 dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, src7, src8, src9
smull \dst1\().4s, \src1\().4h, v0.h[0]
smull2 \dst2\().4s, \src1\().8h, v0.h[0]
smull \dst3\().4s, \src2\().4h, v0.h[0]
smull2 \dst4\().4s, \src2\().8h, v0.h[0]
smlal \dst1\().4s, \src2\().4h, v0.h[1]
smlal2 \dst2\().4s, \src2\().8h, v0.h[1]
smlal \dst3\().4s, \src3\().4h, v0.h[1]
smlal2 \dst4\().4s, \src3\().8h, v0.h[1]
smlal \dst1\().4s, \src3\().4h, v0.h[2]
smlal2 \dst2\().4s, \src3\().8h, v0.h[2]
smlal \dst3\().4s, \src4\().4h, v0.h[2]
smlal2 \dst4\().4s, \src4\().8h, v0.h[2]
smlal \dst1\().4s, \src4\().4h, v0.h[3]
smlal2 \dst2\().4s, \src4\().8h, v0.h[3]
smlal \dst3\().4s, \src5\().4h, v0.h[3]
smlal2 \dst4\().4s, \src5\().8h, v0.h[3]
smlal \dst1\().4s, \src5\().4h, v0.h[4]
smlal2 \dst2\().4s, \src5\().8h, v0.h[4]
smlal \dst3\().4s, \src6\().4h, v0.h[4]
smlal2 \dst4\().4s, \src6\().8h, v0.h[4]
smlal \dst1\().4s, \src6\().4h, v0.h[5]
smlal2 \dst2\().4s, \src6\().8h, v0.h[5]
smlal \dst3\().4s, \src7\().4h, v0.h[5]
smlal2 \dst4\().4s, \src7\().8h, v0.h[5]
smlal \dst1\().4s, \src7\().4h, v0.h[6]
smlal2 \dst2\().4s, \src7\().8h, v0.h[6]
smlal \dst3\().4s, \src8\().4h, v0.h[6]
smlal2 \dst4\().4s, \src8\().8h, v0.h[6]
smlal \dst1\().4s, \src8\().4h, v0.h[7]
smlal2 \dst2\().4s, \src8\().8h, v0.h[7]
smlal \dst3\().4s, \src9\().4h, v0.h[7]
smlal2 \dst4\().4s, \src9\().8h, v0.h[7]
.endm
// Instantiate a vertical filter function for filtering 8 pixels at a time.
// The height is passed in x4, the width in x5 and the filter coefficients
// in x6.
.macro do_8tap_8v type
function \type\()_8tap_8v
sub x2, x2, x3, lsl #1
sub x2, x2, x3
ld1 {v0.8h}, [x6]
1:
.ifc \type,avg
mov x7, x0
.endif
mov x6, x4
ld1 {v17.8h}, [x2], x3
ld1 {v18.8h}, [x2], x3
ld1 {v19.8h}, [x2], x3
ld1 {v20.8h}, [x2], x3
ld1 {v21.8h}, [x2], x3
ld1 {v22.8h}, [x2], x3
ld1 {v23.8h}, [x2], x3
2:
ld1 {v24.8h}, [x2], x3
ld1 {v25.8h}, [x2], x3
ld1 {v26.8h}, [x2], x3
ld1 {v27.8h}, [x2], x3
convolve8 v2, v3, v4, v5, v17, v18, v19, v20, v21, v22, v23, v24, v25
convolve8 v6, v7, v30, v31, v19, v20, v21, v22, v23, v24, v25, v26, v27
do_store8 v2, v3, v4, v5, v6, v7, v30, v31, v1, \type
subs x6, x6, #4
b.eq 8f
ld1 {v16.8h}, [x2], x3
ld1 {v17.8h}, [x2], x3
ld1 {v18.8h}, [x2], x3
ld1 {v19.8h}, [x2], x3
convolve8 v2, v3, v4, v5, v21, v22, v23, v24, v25, v26, v27, v16, v17
convolve8 v6, v7, v20, v21, v23, v24, v25, v26, v27, v16, v17, v18, v19
do_store8 v2, v3, v4, v5, v6, v7, v20, v21, v1, \type
subs x6, x6, #4
b.eq 8f
ld1 {v20.8h}, [x2], x3
ld1 {v21.8h}, [x2], x3
ld1 {v22.8h}, [x2], x3
ld1 {v23.8h}, [x2], x3
convolve8 v2, v3, v4, v5, v25, v26, v27, v16, v17, v18, v19, v20, v21
convolve8 v6, v7, v24, v25, v27, v16, v17, v18, v19, v20, v21, v22, v23
do_store8 v2, v3, v4, v5, v6, v7, v24, v25, v1, \type
subs x6, x6, #4
b.ne 2b
8:
subs x5, x5, #8
b.eq 9f
// x0 -= h * dst_stride
msub x0, x1, x4, x0
// x2 -= h * src_stride
msub x2, x3, x4, x2
// x2 -= 8 * src_stride
sub x2, x2, x3, lsl #3
// x2 += 1 * src_stride
add x2, x2, x3
add x2, x2, #16
add x0, x0, #16
b 1b
9:
ret
endfunc
.endm
do_8tap_8v put
do_8tap_8v avg
// Instantiate a vertical filter function for filtering a 4 pixels wide
// slice. This only is designed to work for 4 or 8 output lines.
.macro do_8tap_4v type
function \type\()_8tap_4v
sub x2, x2, x3, lsl #1
sub x2, x2, x3
ld1 {v0.8h}, [x6]
.ifc \type,avg
mov x7, x0
.endif
ld1 {v16.4h}, [x2], x3
ld1 {v17.4h}, [x2], x3
ld1 {v18.4h}, [x2], x3
ld1 {v19.4h}, [x2], x3
ld1 {v20.4h}, [x2], x3
ld1 {v21.4h}, [x2], x3
ld1 {v22.4h}, [x2], x3
ld1 {v23.4h}, [x2], x3
ld1 {v24.4h}, [x2], x3
ld1 {v25.4h}, [x2], x3
ld1 {v26.4h}, [x2], x3
convolve4 v2, v3, v16, v17, v18, v19, v20, v21, v22, v23, v24, v30, v31
convolve4 v4, v5, v18, v19, v20, v21, v22, v23, v24, v25, v26, v30, v31
do_store4 v2, v3, v4, v5, v28, v29, v30, v31, v1, \type
subs x4, x4, #4
b.eq 9f
ld1 {v27.4h}, [x2], x3
ld1 {v28.4h}, [x2], x3
ld1 {v29.4h}, [x2], x3
ld1 {v30.4h}, [x2], x3
convolve4 v2, v3, v20, v21, v22, v23, v24, v25, v26, v27, v28, v16, v17
convolve4 v4, v5, v22, v23, v24, v25, v26, v27, v28, v29, v30, v16, v17
do_store4 v2, v3, v4, v5, v16, v17, v18, v19, v1, \type
9:
ret
endfunc
.endm
do_8tap_4v put
do_8tap_4v avg
.macro do_8tap_v_func type, filter, offset, size, bpp
function ff_vp9_\type\()_\filter\()\size\()_v_\bpp\()_neon, export=1
uxtw x4, w4
mvni v1.8h, #((0xff << (\bpp - 8)) & 0xff), lsl #8
movrel x5, X(ff_vp9_subpel_filters), 256*\offset
add x6, x5, w6, uxtw #4
mov x5, #\size
.if \size >= 8
b \type\()_8tap_8v
.else
b \type\()_8tap_4v
.endif
endfunc
.endm
.macro do_8tap_v_filters size, bpp
do_8tap_v_func put, regular, 1, \size, \bpp
do_8tap_v_func avg, regular, 1, \size, \bpp
do_8tap_v_func put, sharp, 2, \size, \bpp
do_8tap_v_func avg, sharp, 2, \size, \bpp
do_8tap_v_func put, smooth, 0, \size, \bpp
do_8tap_v_func avg, smooth, 0, \size, \bpp
.endm
.macro do_8tap_v_filters_bpp bpp
do_8tap_v_filters 64, \bpp
do_8tap_v_filters 32, \bpp
do_8tap_v_filters 16, \bpp
do_8tap_v_filters 8, \bpp
do_8tap_v_filters 4, \bpp
.endm
do_8tap_v_filters_bpp 10
do_8tap_v_filters_bpp 12
+81
View File
@@ -0,0 +1,81 @@
/*
* Copyright (c) 2016 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
// All public functions in this file have the following signature:
// typedef void (*vp9_mc_func)(uint8_t *dst, ptrdiff_t dst_stride,
// const uint8_t *ref, ptrdiff_t ref_stride,
// int h, int mx, int my);
function ff_vp9_copy128_aarch64, export=1
1:
ldp x5, x6, [x2]
ldp x7, x8, [x2, #16]
stp x5, x6, [x0]
ldp x9, x10, [x2, #32]
stp x7, x8, [x0, #16]
subs w4, w4, #1
ldp x11, x12, [x2, #48]
stp x9, x10, [x0, #32]
stp x11, x12, [x0, #48]
ldp x5, x6, [x2, #64]
ldp x7, x8, [x2, #80]
stp x5, x6, [x0, #64]
ldp x9, x10, [x2, #96]
stp x7, x8, [x0, #80]
ldp x11, x12, [x2, #112]
stp x9, x10, [x0, #96]
stp x11, x12, [x0, #112]
add x2, x2, x3
add x0, x0, x1
b.ne 1b
ret
endfunc
function ff_vp9_copy64_aarch64, export=1
1:
ldp x5, x6, [x2]
ldp x7, x8, [x2, #16]
stp x5, x6, [x0]
ldp x9, x10, [x2, #32]
stp x7, x8, [x0, #16]
subs w4, w4, #1
ldp x11, x12, [x2, #48]
stp x9, x10, [x0, #32]
stp x11, x12, [x0, #48]
add x2, x2, x3
add x0, x0, x1
b.ne 1b
ret
endfunc
function ff_vp9_copy32_aarch64, export=1
1:
ldp x5, x6, [x2]
ldp x7, x8, [x2, #16]
stp x5, x6, [x0]
subs w4, w4, #1
stp x7, x8, [x0, #16]
add x2, x2, x3
add x0, x0, x1
b.ne 1b
ret
endfunc
+665
View File
@@ -0,0 +1,665 @@
/*
* Copyright (c) 2016 Google Inc.
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
// All public functions in this file have the following signature:
// typedef void (*vp9_mc_func)(uint8_t *dst, ptrdiff_t dst_stride,
// const uint8_t *ref, ptrdiff_t ref_stride,
// int h, int mx, int my);
function ff_vp9_avg64_neon, export=1
mov x5, x0
1:
ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x2], x3
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x2], x3
urhadd v0.16b, v0.16b, v4.16b
urhadd v1.16b, v1.16b, v5.16b
ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x0], x1
urhadd v2.16b, v2.16b, v6.16b
urhadd v3.16b, v3.16b, v7.16b
subs w4, w4, #2
urhadd v16.16b, v16.16b, v20.16b
urhadd v17.16b, v17.16b, v21.16b
st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], x1
urhadd v18.16b, v18.16b, v22.16b
urhadd v19.16b, v19.16b, v23.16b
st1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg32_neon, export=1
1:
ld1 {v2.16b, v3.16b}, [x2], x3
ld1 {v0.16b, v1.16b}, [x0]
urhadd v0.16b, v0.16b, v2.16b
urhadd v1.16b, v1.16b, v3.16b
subs w4, w4, #1
st1 {v0.16b, v1.16b}, [x0], x1
b.ne 1b
ret
endfunc
function ff_vp9_copy16_neon, export=1
add x5, x0, x1
lsl x1, x1, #1
add x6, x2, x3
lsl x3, x3, #1
1:
ld1 {v0.16b}, [x2], x3
ld1 {v1.16b}, [x6], x3
ld1 {v2.16b}, [x2], x3
ld1 {v3.16b}, [x6], x3
subs w4, w4, #4
st1 {v0.16b}, [x0], x1
st1 {v1.16b}, [x5], x1
st1 {v2.16b}, [x0], x1
st1 {v3.16b}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg16_neon, export=1
mov x5, x0
1:
ld1 {v2.16b}, [x2], x3
ld1 {v0.16b}, [x0], x1
ld1 {v3.16b}, [x2], x3
urhadd v0.16b, v0.16b, v2.16b
ld1 {v1.16b}, [x0], x1
urhadd v1.16b, v1.16b, v3.16b
subs w4, w4, #2
st1 {v0.16b}, [x5], x1
st1 {v1.16b}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_copy8_neon, export=1
1:
ld1 {v0.8b}, [x2], x3
ld1 {v1.8b}, [x2], x3
subs w4, w4, #2
st1 {v0.8b}, [x0], x1
st1 {v1.8b}, [x0], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg8_neon, export=1
mov x5, x0
1:
ld1 {v2.8b}, [x2], x3
ld1 {v0.8b}, [x0], x1
ld1 {v3.8b}, [x2], x3
urhadd v0.8b, v0.8b, v2.8b
ld1 {v1.8b}, [x0], x1
urhadd v1.8b, v1.8b, v3.8b
subs w4, w4, #2
st1 {v0.8b}, [x5], x1
st1 {v1.8b}, [x5], x1
b.ne 1b
ret
endfunc
function ff_vp9_copy4_neon, export=1
1:
ld1 {v0.s}[0], [x2], x3
ld1 {v1.s}[0], [x2], x3
st1 {v0.s}[0], [x0], x1
ld1 {v2.s}[0], [x2], x3
st1 {v1.s}[0], [x0], x1
ld1 {v3.s}[0], [x2], x3
subs w4, w4, #4
st1 {v2.s}[0], [x0], x1
st1 {v3.s}[0], [x0], x1
b.ne 1b
ret
endfunc
function ff_vp9_avg4_neon, export=1
mov x5, x0
1:
ld1 {v2.s}[0], [x2], x3
ld1 {v0.s}[0], [x0], x1
ld1 {v2.s}[1], [x2], x3
ld1 {v0.s}[1], [x0], x1
ld1 {v3.s}[0], [x2], x3
ld1 {v1.s}[0], [x0], x1
ld1 {v3.s}[1], [x2], x3
ld1 {v1.s}[1], [x0], x1
subs w4, w4, #4
urhadd v0.8b, v0.8b, v2.8b
urhadd v1.8b, v1.8b, v3.8b
st1 {v0.s}[0], [x5], x1
st1 {v0.s}[1], [x5], x1
st1 {v1.s}[0], [x5], x1
st1 {v1.s}[1], [x5], x1
b.ne 1b
ret
endfunc
// Extract a vector from src1-src2 and src4-src5 (src1-src3 and src4-src6
// for size >= 16), and multiply-accumulate into dst1 and dst3 (or
// dst1-dst2 and dst3-dst4 for size >= 16)
.macro extmla dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, offset, size
ext v20.16b, \src1\().16b, \src2\().16b, #(2*\offset)
ext v22.16b, \src4\().16b, \src5\().16b, #(2*\offset)
.if \size >= 16
mla \dst1\().8h, v20.8h, v0.h[\offset]
ext v21.16b, \src2\().16b, \src3\().16b, #(2*\offset)
mla \dst3\().8h, v22.8h, v0.h[\offset]
ext v23.16b, \src5\().16b, \src6\().16b, #(2*\offset)
mla \dst2\().8h, v21.8h, v0.h[\offset]
mla \dst4\().8h, v23.8h, v0.h[\offset]
.elseif \size == 8
mla \dst1\().8h, v20.8h, v0.h[\offset]
mla \dst3\().8h, v22.8h, v0.h[\offset]
.else
mla \dst1\().4h, v20.4h, v0.h[\offset]
mla \dst3\().4h, v22.4h, v0.h[\offset]
.endif
.endm
// The same as above, but don't accumulate straight into the
// destination, but use a temp register and accumulate with saturation.
.macro extmulqadd dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, offset, size
ext v20.16b, \src1\().16b, \src2\().16b, #(2*\offset)
ext v22.16b, \src4\().16b, \src5\().16b, #(2*\offset)
.if \size >= 16
mul v20.8h, v20.8h, v0.h[\offset]
ext v21.16b, \src2\().16b, \src3\().16b, #(2*\offset)
mul v22.8h, v22.8h, v0.h[\offset]
ext v23.16b, \src5\().16b, \src6\().16b, #(2*\offset)
mul v21.8h, v21.8h, v0.h[\offset]
mul v23.8h, v23.8h, v0.h[\offset]
.elseif \size == 8
mul v20.8h, v20.8h, v0.h[\offset]
mul v22.8h, v22.8h, v0.h[\offset]
.else
mul v20.4h, v20.4h, v0.h[\offset]
mul v22.4h, v22.4h, v0.h[\offset]
.endif
.if \size == 4
sqadd \dst1\().4h, \dst1\().4h, v20.4h
sqadd \dst3\().4h, \dst3\().4h, v22.4h
.else
sqadd \dst1\().8h, \dst1\().8h, v20.8h
sqadd \dst3\().8h, \dst3\().8h, v22.8h
.if \size >= 16
sqadd \dst2\().8h, \dst2\().8h, v21.8h
sqadd \dst4\().8h, \dst4\().8h, v23.8h
.endif
.endif
.endm
// Instantiate a horizontal filter function for the given size.
// This can work on 4, 8 or 16 pixels in parallel; for larger
// widths it will do 16 pixels at a time and loop horizontally.
// The actual width is passed in x5, the height in w4 and the
// filter coefficients in x9. idx2 is the index of the largest
// filter coefficient (3 or 4) and idx1 is the other one of them.
.macro do_8tap_h type, size, idx1, idx2
function \type\()_8tap_\size\()h_\idx1\idx2
sub x2, x2, #3
add x6, x0, x1
add x7, x2, x3
add x1, x1, x1
add x3, x3, x3
// Only size >= 16 loops horizontally and needs
// reduced dst stride
.if \size >= 16
sub x1, x1, x5
.elseif \size == 4
add x12, x2, #8
add x13, x7, #8
.endif
// size >= 16 loads two qwords and increments x2,
// for size 4/8 it's enough with one qword and no
// postincrement
.if \size >= 16
sub x3, x3, x5
sub x3, x3, #8
.endif
// Load the filter vector
ld1 {v0.8h}, [x9]
1:
.if \size >= 16
mov x9, x5
.endif
// Load src
.if \size >= 16
ld1 {v4.8b, v5.8b, v6.8b}, [x2], #24
ld1 {v16.8b, v17.8b, v18.8b}, [x7], #24
.elseif \size == 8
ld1 {v4.8b, v5.8b}, [x2]
ld1 {v16.8b, v17.8b}, [x7]
.else // \size == 4
ld1 {v4.8b}, [x2]
ld1 {v16.8b}, [x7]
ld1 {v5.s}[0], [x12], x3
ld1 {v17.s}[0], [x13], x3
.endif
uxtl v4.8h, v4.8b
uxtl v5.8h, v5.8b
uxtl v16.8h, v16.8b
uxtl v17.8h, v17.8b
.if \size >= 16
uxtl v6.8h, v6.8b
uxtl v18.8h, v18.8b
.endif
2:
// Accumulate, adding idx2 last with a separate
// saturating add. The positive filter coefficients
// for all indices except idx2 must add up to less
// than 127 for this not to overflow.
mul v1.8h, v4.8h, v0.h[0]
mul v24.8h, v16.8h, v0.h[0]
.if \size >= 16
mul v2.8h, v5.8h, v0.h[0]
mul v25.8h, v17.8h, v0.h[0]
.endif
extmla v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, 1, \size
extmla v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, 2, \size
extmla v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, \idx1, \size
extmla v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, 5, \size
extmla v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, 6, \size
extmla v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, 7, \size
extmulqadd v1, v2, v24, v25, v4, v5, v6, v16, v17, v18, \idx2, \size
// Round, shift and saturate
sqrshrun v1.8b, v1.8h, #7
sqrshrun v24.8b, v24.8h, #7
.if \size >= 16
sqrshrun2 v1.16b, v2.8h, #7
sqrshrun2 v24.16b, v25.8h, #7
.endif
// Average
.ifc \type,avg
.if \size >= 16
ld1 {v2.16b}, [x0]
ld1 {v3.16b}, [x6]
urhadd v1.16b, v1.16b, v2.16b
urhadd v24.16b, v24.16b, v3.16b
.elseif \size == 8
ld1 {v2.8b}, [x0]
ld1 {v3.8b}, [x6]
urhadd v1.8b, v1.8b, v2.8b
urhadd v24.8b, v24.8b, v3.8b
.else
ld1 {v2.s}[0], [x0]
ld1 {v3.s}[0], [x6]
urhadd v1.8b, v1.8b, v2.8b
urhadd v24.8b, v24.8b, v3.8b
.endif
.endif
// Store and loop horizontally (for size >= 16)
.if \size >= 16
subs x9, x9, #16
st1 {v1.16b}, [x0], #16
st1 {v24.16b}, [x6], #16
b.eq 3f
mov v4.16b, v6.16b
mov v16.16b, v18.16b
ld1 {v6.16b}, [x2], #16
ld1 {v18.16b}, [x7], #16
uxtl v5.8h, v6.8b
uxtl2 v6.8h, v6.16b
uxtl v17.8h, v18.8b
uxtl2 v18.8h, v18.16b
b 2b
.elseif \size == 8
st1 {v1.8b}, [x0]
st1 {v24.8b}, [x6]
.else // \size == 4
st1 {v1.s}[0], [x0]
st1 {v24.s}[0], [x6]
.endif
3:
// Loop vertically
add x0, x0, x1
add x6, x6, x1
add x2, x2, x3
add x7, x7, x3
subs w4, w4, #2
b.ne 1b
ret
endfunc
.endm
.macro do_8tap_h_size size
do_8tap_h put, \size, 3, 4
do_8tap_h avg, \size, 3, 4
do_8tap_h put, \size, 4, 3
do_8tap_h avg, \size, 4, 3
.endm
do_8tap_h_size 4
do_8tap_h_size 8
do_8tap_h_size 16
.macro do_8tap_h_func type, filter, offset, size
function ff_vp9_\type\()_\filter\()\size\()_h_neon, export=1
movrel x6, X(ff_vp9_subpel_filters), 256*\offset
cmp w5, #8
add x9, x6, w5, uxtw #4
mov x5, #\size
.if \size >= 16
b.ge \type\()_8tap_16h_34
b \type\()_8tap_16h_43
.else
b.ge \type\()_8tap_\size\()h_34
b \type\()_8tap_\size\()h_43
.endif
endfunc
.endm
.macro do_8tap_h_filters size
do_8tap_h_func put, regular, 1, \size
do_8tap_h_func avg, regular, 1, \size
do_8tap_h_func put, sharp, 2, \size
do_8tap_h_func avg, sharp, 2, \size
do_8tap_h_func put, smooth, 0, \size
do_8tap_h_func avg, smooth, 0, \size
.endm
do_8tap_h_filters 64
do_8tap_h_filters 32
do_8tap_h_filters 16
do_8tap_h_filters 8
do_8tap_h_filters 4
// Vertical filters
// Round, shift and saturate and store reg1-reg2 over 4 lines
.macro do_store4 reg1, reg2, tmp1, tmp2, type
sqrshrun \reg1\().8b, \reg1\().8h, #7
sqrshrun \reg2\().8b, \reg2\().8h, #7
.ifc \type,avg
ld1 {\tmp1\().s}[0], [x7], x1
ld1 {\tmp2\().s}[0], [x7], x1
ld1 {\tmp1\().s}[1], [x7], x1
ld1 {\tmp2\().s}[1], [x7], x1
urhadd \reg1\().8b, \reg1\().8b, \tmp1\().8b
urhadd \reg2\().8b, \reg2\().8b, \tmp2\().8b
.endif
st1 {\reg1\().s}[0], [x0], x1
st1 {\reg2\().s}[0], [x0], x1
st1 {\reg1\().s}[1], [x0], x1
st1 {\reg2\().s}[1], [x0], x1
.endm
// Round, shift and saturate and store reg1-4
.macro do_store reg1, reg2, reg3, reg4, tmp1, tmp2, tmp3, tmp4, type
sqrshrun \reg1\().8b, \reg1\().8h, #7
sqrshrun \reg2\().8b, \reg2\().8h, #7
sqrshrun \reg3\().8b, \reg3\().8h, #7
sqrshrun \reg4\().8b, \reg4\().8h, #7
.ifc \type,avg
ld1 {\tmp1\().8b}, [x7], x1
ld1 {\tmp2\().8b}, [x7], x1
ld1 {\tmp3\().8b}, [x7], x1
ld1 {\tmp4\().8b}, [x7], x1
urhadd \reg1\().8b, \reg1\().8b, \tmp1\().8b
urhadd \reg2\().8b, \reg2\().8b, \tmp2\().8b
urhadd \reg3\().8b, \reg3\().8b, \tmp3\().8b
urhadd \reg4\().8b, \reg4\().8b, \tmp4\().8b
.endif
st1 {\reg1\().8b}, [x0], x1
st1 {\reg2\().8b}, [x0], x1
st1 {\reg3\().8b}, [x0], x1
st1 {\reg4\().8b}, [x0], x1
.endm
// Evaluate the filter twice in parallel, from the inputs src1-src9 into dst1-dst2
// (src1-src8 into dst1, src2-src9 into dst2), adding idx2 separately
// at the end with saturation. Indices 0 and 7 always have negative or zero
// coefficients, so they can be accumulated into tmp1-tmp2 together with the
// largest coefficient.
.macro convolve dst1, dst2, src1, src2, src3, src4, src5, src6, src7, src8, src9, idx1, idx2, tmp1, tmp2
mul \dst1\().8h, \src2\().8h, v0.h[1]
mul \dst2\().8h, \src3\().8h, v0.h[1]
mul \tmp1\().8h, \src1\().8h, v0.h[0]
mul \tmp2\().8h, \src2\().8h, v0.h[0]
mla \dst1\().8h, \src3\().8h, v0.h[2]
mla \dst2\().8h, \src4\().8h, v0.h[2]
.if \idx1 == 3
mla \dst1\().8h, \src4\().8h, v0.h[3]
mla \dst2\().8h, \src5\().8h, v0.h[3]
.else
mla \dst1\().8h, \src5\().8h, v0.h[4]
mla \dst2\().8h, \src6\().8h, v0.h[4]
.endif
mla \dst1\().8h, \src6\().8h, v0.h[5]
mla \dst2\().8h, \src7\().8h, v0.h[5]
mla \tmp1\().8h, \src8\().8h, v0.h[7]
mla \tmp2\().8h, \src9\().8h, v0.h[7]
mla \dst1\().8h, \src7\().8h, v0.h[6]
mla \dst2\().8h, \src8\().8h, v0.h[6]
.if \idx2 == 3
mla \tmp1\().8h, \src4\().8h, v0.h[3]
mla \tmp2\().8h, \src5\().8h, v0.h[3]
.else
mla \tmp1\().8h, \src5\().8h, v0.h[4]
mla \tmp2\().8h, \src6\().8h, v0.h[4]
.endif
sqadd \dst1\().8h, \dst1\().8h, \tmp1\().8h
sqadd \dst2\().8h, \dst2\().8h, \tmp2\().8h
.endm
// Load pixels and extend them to 16 bit
.macro loadl dst1, dst2, dst3, dst4
ld1 {v1.8b}, [x2], x3
ld1 {v2.8b}, [x2], x3
ld1 {v3.8b}, [x2], x3
.ifnb \dst4
ld1 {v4.8b}, [x2], x3
.endif
uxtl \dst1\().8h, v1.8b
uxtl \dst2\().8h, v2.8b
uxtl \dst3\().8h, v3.8b
.ifnb \dst4
uxtl \dst4\().8h, v4.8b
.endif
.endm
// Instantiate a vertical filter function for filtering 8 pixels at a time.
// The height is passed in x4, the width in x5 and the filter coefficients
// in x6. idx2 is the index of the largest filter coefficient (3 or 4)
// and idx1 is the other one of them.
.macro do_8tap_8v type, idx1, idx2
function \type\()_8tap_8v_\idx1\idx2
sub x2, x2, x3, lsl #1
sub x2, x2, x3
ld1 {v0.8h}, [x6]
1:
.ifc \type,avg
mov x7, x0
.endif
mov x6, x4
loadl v17, v18, v19
loadl v20, v21, v22, v23
2:
loadl v24, v25, v26, v27
convolve v1, v2, v17, v18, v19, v20, v21, v22, v23, v24, v25, \idx1, \idx2, v5, v6
convolve v3, v4, v19, v20, v21, v22, v23, v24, v25, v26, v27, \idx1, \idx2, v5, v6
do_store v1, v2, v3, v4, v5, v6, v7, v28, \type
subs x6, x6, #4
b.eq 8f
loadl v16, v17, v18, v19
convolve v1, v2, v21, v22, v23, v24, v25, v26, v27, v16, v17, \idx1, \idx2, v5, v6
convolve v3, v4, v23, v24, v25, v26, v27, v16, v17, v18, v19, \idx1, \idx2, v5, v6
do_store v1, v2, v3, v4, v5, v6, v7, v28, \type
subs x6, x6, #4
b.eq 8f
loadl v20, v21, v22, v23
convolve v1, v2, v25, v26, v27, v16, v17, v18, v19, v20, v21, \idx1, \idx2, v5, v6
convolve v3, v4, v27, v16, v17, v18, v19, v20, v21, v22, v23, \idx1, \idx2, v5, v6
do_store v1, v2, v3, v4, v5, v6, v7, v28, \type
subs x6, x6, #4
b.ne 2b
8:
subs x5, x5, #8
b.eq 9f
// x0 -= h * dst_stride
msub x0, x1, x4, x0
// x2 -= h * src_stride
msub x2, x3, x4, x2
// x2 -= 8 * src_stride
sub x2, x2, x3, lsl #3
// x2 += 1 * src_stride
add x2, x2, x3
add x2, x2, #8
add x0, x0, #8
b 1b
9:
ret
endfunc
.endm
do_8tap_8v put, 3, 4
do_8tap_8v put, 4, 3
do_8tap_8v avg, 3, 4
do_8tap_8v avg, 4, 3
// Instantiate a vertical filter function for filtering a 4 pixels wide
// slice. The first half of the registers contain one row, while the second
// half of a register contains the second-next row (also stored in the first
// half of the register two steps ahead). The convolution does two outputs
// at a time; the output of v17-v24 into one, and v18-v25 into another one.
// The first half of first output is the first output row, the first half
// of the other output is the second output row. The second halves of the
// registers are rows 3 and 4.
// This only is designed to work for 4 or 8 output lines.
.macro do_8tap_4v type, idx1, idx2
function \type\()_8tap_4v_\idx1\idx2
sub x2, x2, x3, lsl #1
sub x2, x2, x3
ld1 {v0.8h}, [x6]
.ifc \type,avg
mov x7, x0
.endif
ld1 {v1.s}[0], [x2], x3
ld1 {v2.s}[0], [x2], x3
ld1 {v3.s}[0], [x2], x3
ld1 {v4.s}[0], [x2], x3
ld1 {v5.s}[0], [x2], x3
ld1 {v6.s}[0], [x2], x3
trn1 v1.2s, v1.2s, v3.2s
ld1 {v7.s}[0], [x2], x3
trn1 v2.2s, v2.2s, v4.2s
ld1 {v26.s}[0], [x2], x3
uxtl v17.8h, v1.8b
trn1 v3.2s, v3.2s, v5.2s
ld1 {v27.s}[0], [x2], x3
uxtl v18.8h, v2.8b
trn1 v4.2s, v4.2s, v6.2s
ld1 {v28.s}[0], [x2], x3
uxtl v19.8h, v3.8b
trn1 v5.2s, v5.2s, v7.2s
ld1 {v29.s}[0], [x2], x3
uxtl v20.8h, v4.8b
trn1 v6.2s, v6.2s, v26.2s
uxtl v21.8h, v5.8b
trn1 v7.2s, v7.2s, v27.2s
uxtl v22.8h, v6.8b
trn1 v26.2s, v26.2s, v28.2s
uxtl v23.8h, v7.8b
trn1 v27.2s, v27.2s, v29.2s
uxtl v24.8h, v26.8b
uxtl v25.8h, v27.8b
convolve v1, v2, v17, v18, v19, v20, v21, v22, v23, v24, v25, \idx1, \idx2, v3, v4
do_store4 v1, v2, v5, v6, \type
subs x4, x4, #4
b.eq 9f
ld1 {v1.s}[0], [x2], x3
ld1 {v2.s}[0], [x2], x3
trn1 v28.2s, v28.2s, v1.2s
trn1 v29.2s, v29.2s, v2.2s
ld1 {v1.s}[1], [x2], x3
uxtl v26.8h, v28.8b
ld1 {v2.s}[1], [x2], x3
uxtl v27.8h, v29.8b
uxtl v28.8h, v1.8b
uxtl v29.8h, v2.8b
convolve v1, v2, v21, v22, v23, v24, v25, v26, v27, v28, v29, \idx1, \idx2, v3, v4
do_store4 v1, v2, v5, v6, \type
9:
ret
endfunc
.endm
do_8tap_4v put, 3, 4
do_8tap_4v put, 4, 3
do_8tap_4v avg, 3, 4
do_8tap_4v avg, 4, 3
.macro do_8tap_v_func type, filter, offset, size
function ff_vp9_\type\()_\filter\()\size\()_v_neon, export=1
uxtw x4, w4
movrel x5, X(ff_vp9_subpel_filters), 256*\offset
cmp w6, #8
add x6, x5, w6, uxtw #4
mov x5, #\size
.if \size >= 8
b.ge \type\()_8tap_8v_34
b \type\()_8tap_8v_43
.else
b.ge \type\()_8tap_4v_34
b \type\()_8tap_4v_43
.endif
endfunc
.endm
.macro do_8tap_v_filters size
do_8tap_v_func put, regular, 1, \size
do_8tap_v_func avg, regular, 1, \size
do_8tap_v_func put, sharp, 2, \size
do_8tap_v_func avg, sharp, 2, \size
do_8tap_v_func put, smooth, 0, \size
do_8tap_v_func avg, smooth, 0, \size
.endm
do_8tap_v_filters 64
do_8tap_v_filters 32
do_8tap_v_filters 16
do_8tap_v_filters 8
do_8tap_v_filters 4
+10
View File
@@ -0,0 +1,10 @@
clean::
$(RM) $(CLEANSUFFIXES:%=libavcodec/aarch64/vvc/%)
OBJS-$(CONFIG_VVC_DECODER) += aarch64/vvc/dsp_init.o
NEON-OBJS-$(CONFIG_VVC_DECODER) += aarch64/vvc/alf.o \
aarch64/vvc/inter.o \
aarch64/vvc/sad.o \
aarch64/h26x/epel_neon.o \
aarch64/h26x/qpel_neon.o \
aarch64/h26x/sao_neon.o
+293
View File
@@ -0,0 +1,293 @@
/*
* Copyright (c) 2024 Zhao Zhili <quinkblack@foxmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
.macro alf_luma_filter_pixel index, pix_size, addr1, addr2, offset1, offset2
.if \pix_size == 1
ldur d3, [\addr1, #\offset1]
ldur d4, [\addr2, #\offset2]
uxtl v6.8h, v3.8b
uxtl v7.8h, v4.8b
.else
ldur q6, [\addr1, #(2*\offset1)]
ldur q7, [\addr2, #(2*\offset2)]
.endif
.if \index < 8
dup v17.4h, v0.h[\index] // clip
dup v18.4h, v16.h[\index] // -clip
dup v19.4h, v1.h[\index] // filter
dup v26.4h, v22.h[\index] // clip
dup v27.4h, v23.h[\index] // -clip
dup v28.4h, v24.h[\index] // filter
.else
dup v17.4h, v0.h[\index - 8] // clip
dup v18.4h, v16.h[\index - 8] // -clip
dup v19.4h, v1.h[\index - 8] // filter
dup v26.4h, v22.h[\index - 8] // clip
dup v27.4h, v23.h[\index - 8] // -clip
dup v28.4h, v24.h[\index - 8] // filter
.endif
ins v17.d[1], v26.d[0]
ins v18.d[1], v27.d[0]
ins v19.d[1], v28.d[0]
sub v6.8h, v6.8h, v5.8h
sub v7.8h, v7.8h, v5.8h
smin v6.8h, v6.8h, v17.8h
smin v7.8h, v7.8h, v17.8h
smax v6.8h, v6.8h, v18.8h
smax v7.8h, v7.8h, v18.8h
add v6.8h, v6.8h, v7.8h
smlal v20.4s, v19.4h, v6.4h // v20: sum
smlal2 v21.4s, v19.8h, v6.8h // v21: sum
.endm
/* x0: dst
* x1: pp
* x2: filter
* x3: clip
* w4: is_near_vb
* w5: pix_max
*/
.macro alf_filter_luma_kernel, pix_size
dst .req x0
pp .req x1
filter .req x2
clip .req x3
is_near_vb .req w4
pix_max .req w5
.if \pix_size > 1
dup v25.8h, pix_max // pix_max
.endif
ldr q0, [clip] // clip
ldr q1, [filter] // filter
ldur q22, [clip, #24] // clip
ldur q24, [filter, #24] // filter
ldr x5, [pp] // x5: p0
ldr x6, [pp, #(5*8)] // x6: p5
ldr x7, [pp, #(6*8)] // x7: p6
neg v16.8h, v0.8h // -clip
neg v23.8h, v22.8h // -clip
.if \pix_size == 1
ldr d2, [x5] // curr
.else
ldr q5, [x5] // curr
.endif
movi v20.4s, #64
cbz is_near_vb, 1f
shl v20.4s, v20.4s, #3
1:
.if \pix_size == 1
uxtl v5.8h, v2.8b
.endif
mov v21.16b, v20.16b
ldr x8, [pp, #(3*8)] // p3
ldr x9, [pp, #(4*8)] // p4
alf_luma_filter_pixel 0, \pix_size, x6, x7, 0, 0
ldr x6, [pp, #(1*8)] // p1
ldr x7, [pp, #(2*8)] // p2
alf_luma_filter_pixel 1, \pix_size, x8, x9, 1, -1
alf_luma_filter_pixel 2, \pix_size, x8, x9, 0, 0
alf_luma_filter_pixel 3, \pix_size, x8, x9, -1, 1
alf_luma_filter_pixel 4, \pix_size, x6, x7, 2, -2
alf_luma_filter_pixel 5, \pix_size, x6, x7, 1, -1
alf_luma_filter_pixel 6, \pix_size, x6, x7, 0, 0
alf_luma_filter_pixel 7, \pix_size, x6, x7, -1, 1
ldr d0, [clip, #16] // clip
ldr d1, [filter, #16] // filter
neg v16.4h, v0.4h // -clip
ldr d22, [clip, #40] // clip
ldr d24, [filter, #40] // filter
neg v23.4h, v22.4h // -clip
alf_luma_filter_pixel 8, \pix_size, x6, x7, -2, 2
alf_luma_filter_pixel 9, \pix_size, x5, x5, 3, -3
alf_luma_filter_pixel 10, \pix_size, x5, x5, 2, -2
alf_luma_filter_pixel 11, \pix_size, x5, x5, 1, -1
cbz is_near_vb, 2f
sshr v20.4s, v20.4s, #10
sshr v21.4s, v21.4s, #10
b 3f
2:
sshr v20.4s, v20.4s, #7
sshr v21.4s, v21.4s, #7
3:
uxtl v22.4s, v5.4h
uxtl2 v23.4s, v5.8h
add v20.4s, v20.4s, v22.4s
add v21.4s, v21.4s, v23.4s
sqxtun v20.4h, v20.4s
sqxtun2 v20.8h, v21.4s
.if \pix_size == 1
sqxtun v20.8b, v20.8h
str d20, [dst]
.else
umin v20.8h, v20.8h, v25.8h
str q20, [dst]
.endif
ret
.unreq dst
.unreq pp
.unreq filter
.unreq clip
.unreq is_near_vb
.unreq pix_max
.endm
.macro alf_chroma_filter_pixel index, pix_size, addr1, addr2, offset1, offset2
.if \pix_size == 1
ldur s3, [\addr1, #\offset1]
ldur s4, [\addr2, #\offset2]
uxtl v6.8h, v3.8b
uxtl v7.8h, v4.8b
.else
ldur d6, [\addr1, #(2*\offset1)]
ldur d7, [\addr2, #(2*\offset2)]
.endif
.if \index < 8
dup v17.4h, v0.h[\index] // v17: clip[0]
dup v18.4h, v16.h[\index] // v18: -clip[0]
dup v19.4h, v1.h[\index] // v19: filter[0]
.else
dup v17.4h, v0.h[\index - 8] // v17: clip[0]
dup v18.4h, v16.h[\index - 8] // v18: -clip[0]
dup v19.4h, v1.h[\index - 8] // v19: filter[0]
.endif
sub v6.4h, v6.4h, v5.4h
sub v7.4h, v7.4h, v5.4h
smin v6.4h, v6.4h, v17.4h
smin v7.4h, v7.4h, v17.4h
smax v6.4h, v6.4h, v18.4h
smax v7.4h, v7.4h, v18.4h
add v6.4h, v6.4h, v7.4h
smlal v20.4s, v19.4h, v6.4h // v20: sum
.endm
/* x0: dst
* x1: pp
* x2: filter
* x3: clip
* w4: is_near_vb
* w5: pix_max
*/
.macro alf_filter_chroma_kernel, pix_size
dst .req x0
pp .req x1
filter .req x2
clip .req x3
is_near_vb .req w4
pix_max .req w5
.if \pix_size > 1
dup v25.4h, pix_max // pix_max
.endif
ldr q0, [clip] // clip
ldr q1, [filter] // filter
ldr x5, [pp] // p0
ldr x6, [pp, #(3*8)] // p3
ldr x7, [pp, #(4*8)] // p4
neg v16.8h, v0.8h // -clip
.if \pix_size == 1
ldr s2, [x5] // curr
.else
ldr d5, [x5] // curr
.endif
movi v20.4s, #64
cbz is_near_vb, 1f
shl v20.4s, v20.4s, #3
1:
.if \pix_size == 1
uxtl v5.8h, v2.8b
.endif
ldr x8, [pp, #(1*8)] // p1
ldr x9, [pp, #(2*8)] // p2
alf_chroma_filter_pixel 0, \pix_size, x6, x7, 0, 0
alf_chroma_filter_pixel 1, \pix_size, x8, x9, 1, -1
alf_chroma_filter_pixel 2, \pix_size, x8, x9, 0, 0
alf_chroma_filter_pixel 3, \pix_size, x8, x9, -1, 1
alf_chroma_filter_pixel 4, \pix_size, x5, x5, 2, -2
alf_chroma_filter_pixel 5, \pix_size, x5, x5, 1, -1
uxtl v22.4s, v5.4h
cbz is_near_vb, 2f
sshr v20.4s, v20.4s, #10
b 3f
2:
sshr v20.4s, v20.4s, #7
3:
add v20.4s, v20.4s, v22.4s
sqxtun v20.4h, v20.4s
.if \pix_size == 1
sqxtun v20.8b, v20.8h
str s20, [dst]
.else
umin v20.4h, v20.4h, v25.4h
str d20, [dst]
.endif
ret
.unreq dst
.unreq pp
.unreq filter
.unreq clip
.unreq is_near_vb
.unreq pix_max
.endm
function ff_alf_filter_luma_kernel_8_neon, export=1
alf_filter_luma_kernel 1
endfunc
function ff_alf_filter_luma_kernel_12_neon, export=1
mov w5, #4095
b 1f
endfunc
function ff_alf_filter_luma_kernel_10_neon, export=1
mov w5, #1023
1:
alf_filter_luma_kernel 2
endfunc
function ff_alf_filter_chroma_kernel_8_neon, export=1
alf_filter_chroma_kernel 1
endfunc
function ff_alf_filter_chroma_kernel_12_neon, export=1
mov w5, #4095
b 1f
endfunc
function ff_alf_filter_chroma_kernel_10_neon, export=1
mov w5, #1023
1:
alf_filter_chroma_kernel 2
endfunc
@@ -0,0 +1,157 @@
/*
* VVC filters DSP
*
* Copyright (C) 2024 Zhao Zhili
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavcodec/bit_depth_template.c"
void FUNC2(ff_alf_filter_luma_kernel, BIT_DEPTH, _neon)(pixel *dst,
const pixel **p,
const int16_t *filter,
const int16_t *clip,
int is_near_vb);
void FUNC2(ff_alf_filter_chroma_kernel, BIT_DEPTH, _neon)(pixel *dst,
const pixel **p,
const int16_t *filter,
const int16_t *clip,
int is_near_vb);
static void FUNC2(alf_filter_luma, BIT_DEPTH, _neon)(uint8_t *_dst,
ptrdiff_t dst_stride,
const uint8_t *_src,
ptrdiff_t src_stride,
const int width, const int height,
const int16_t *filter,
const int16_t *clip,
const int vb_pos)
{
const pixel *src = (pixel *)_src;
dst_stride /= sizeof(pixel);
src_stride /= sizeof(pixel);
for (int y = 0; y < height; y += ALF_BLOCK_SIZE) {
int far = (y + 3 < vb_pos - 3) || (y > vb_pos + 2);
for (int x = 0; x < width; x += 2 * ALF_BLOCK_SIZE) {
const pixel *s0 = src + y * src_stride + x;
const pixel *s1 = s0 + src_stride;
const pixel *s2 = s0 - src_stride;
const pixel *s3 = s1 + src_stride;
const pixel *s4 = s2 - src_stride;
const pixel *s5 = s3 + src_stride;
const pixel *s6 = s4 - src_stride;
for (int i = 0; i < ALF_BLOCK_SIZE; i++) {
pixel *dst = (pixel *) _dst + (y + i) * dst_stride + x;
const pixel *p0 = s0 + i * src_stride;
const pixel *p1 = s1 + i * src_stride;
const pixel *p2 = s2 + i * src_stride;
const pixel *p3 = s3 + i * src_stride;
const pixel *p4 = s4 + i * src_stride;
const pixel *p5 = s5 + i * src_stride;
const pixel *p6 = s6 + i * src_stride;
int is_near_vb = 0;
if (!far) {
is_near_vb = (y + i == vb_pos - 1) || (y + i == vb_pos);
if (is_near_vb) {
p1 = p0;
p2 = p0;
}
if (y + i >= vb_pos - 2 && y + i <= vb_pos + 1) {
p3 = p1;
p4 = p2;
}
if (y + i >= vb_pos - 3 && y + i <= vb_pos + 2) {
p5 = p3;
p6 = p4;
}
}
FUNC2(ff_alf_filter_luma_kernel, BIT_DEPTH, _neon)(dst,
(const pixel *[]) { p0, p1, p2, p3, p4, p5, p6},
filter,
clip,
is_near_vb);
}
filter += 2 * ALF_NUM_COEFF_LUMA;
clip += 2 * ALF_NUM_COEFF_LUMA;
}
}
}
static void FUNC2(alf_filter_chroma, BIT_DEPTH, _neon)(uint8_t *_dst,
ptrdiff_t dst_stride,
const uint8_t *_src,
ptrdiff_t src_stride,
const int width,
const int height,
const int16_t *filter,
const int16_t *clip,
const int vb_pos)
{
const pixel *src = (pixel *)_src;
dst_stride /= sizeof(pixel);
src_stride /= sizeof(pixel);
for (int y = 0; y < height; y += ALF_BLOCK_SIZE) {
int far = (y + 3 < vb_pos - 2) || (y > vb_pos + 1);
for (int x = 0; x < width; x += ALF_BLOCK_SIZE) {
const pixel *s0 = src + y * src_stride + x;
const pixel *s1 = s0 + src_stride;
const pixel *s2 = s0 - src_stride;
const pixel *s3 = s1 + src_stride;
const pixel *s4 = s2 - src_stride;
for (int i = 0; i < ALF_BLOCK_SIZE; i++) {
pixel *dst = (pixel *)_dst + (y + i) * dst_stride + x;
const pixel *p0 = s0 + i * src_stride;
const pixel *p1 = s1 + i * src_stride;
const pixel *p2 = s2 + i * src_stride;
const pixel *p3 = s3 + i * src_stride;
const pixel *p4 = s4 + i * src_stride;
int is_near_vb = 0;
if (!far) {
is_near_vb = (y + i == vb_pos - 1) || (y + i == vb_pos);
if (is_near_vb) {
p1 = p0;
p2 = p0;
}
if (y + i >= vb_pos - 2 && y + i <= vb_pos + 1) {
p3 = p1;
p4 = p2;
}
}
FUNC2(ff_alf_filter_chroma_kernel, BIT_DEPTH, _neon)(dst,
(const pixel *[]){p0, p1, p2, p3, p4},
filter, clip,
is_near_vb);
}
}
}
}
+177
View File
@@ -0,0 +1,177 @@
/*
* VVC filters DSP
*
* Copyright (C) 2024 Zhao Zhili
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/cpu.h"
#include "libavutil/aarch64/cpu.h"
#include "libavcodec/aarch64/h26x/dsp.h"
#include "libavcodec/vvc/dsp.h"
#include "libavcodec/vvc/dec.h"
#include "libavcodec/vvc/ctu.h"
#define BIT_DEPTH 8
#include "alf_template.c"
#undef BIT_DEPTH
#define BIT_DEPTH 10
#include "alf_template.c"
#undef BIT_DEPTH
#define BIT_DEPTH 12
#include "alf_template.c"
#undef BIT_DEPTH
int ff_vvc_sad_neon(const int16_t *src0, const int16_t *src1, int dx, int dy,
const int block_w, const int block_h);
void ff_vvc_avg_8_neon(uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *src0, const int16_t *src1, int width,
int height);
void ff_vvc_avg_10_neon(uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *src0, const int16_t *src1, int width,
int height);
void ff_vvc_avg_12_neon(uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *src0, const int16_t *src1, int width,
int height);
void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd)
{
int cpu_flags = av_get_cpu_flags();
if (!have_neon(cpu_flags))
return;
if (bd == 8) {
c->inter.put[0][1][0][0] = ff_vvc_put_pel_pixels4_8_neon;
c->inter.put[0][2][0][0] = ff_vvc_put_pel_pixels8_8_neon;
c->inter.put[0][3][0][0] = ff_vvc_put_pel_pixels16_8_neon;
c->inter.put[0][4][0][0] = ff_vvc_put_pel_pixels32_8_neon;
c->inter.put[0][5][0][0] = ff_vvc_put_pel_pixels64_8_neon;
c->inter.put[0][6][0][0] = ff_vvc_put_pel_pixels128_8_neon;
c->inter.put[0][1][0][1] = ff_vvc_put_qpel_h4_8_neon;
c->inter.put[0][2][0][1] = ff_vvc_put_qpel_h8_8_neon;
c->inter.put[0][3][0][1] = ff_vvc_put_qpel_h16_8_neon;
c->inter.put[0][4][0][1] =
c->inter.put[0][5][0][1] =
c->inter.put[0][6][0][1] = ff_vvc_put_qpel_h32_8_neon;
c->inter.put[0][1][1][0] = ff_vvc_put_qpel_v4_8_neon;
c->inter.put[0][2][1][0] =
c->inter.put[0][3][1][0] =
c->inter.put[0][4][1][0] =
c->inter.put[0][5][1][0] =
c->inter.put[0][6][1][0] = ff_vvc_put_qpel_v8_8_neon;
c->inter.put[0][1][1][1] = ff_vvc_put_qpel_hv4_8_neon;
c->inter.put[0][2][1][1] = ff_vvc_put_qpel_hv8_8_neon;
c->inter.put[0][3][1][1] = ff_vvc_put_qpel_hv16_8_neon;
c->inter.put[0][4][1][1] = ff_vvc_put_qpel_hv32_8_neon;
c->inter.put[0][5][1][1] = ff_vvc_put_qpel_hv64_8_neon;
c->inter.put[0][6][1][1] = ff_vvc_put_qpel_hv128_8_neon;
c->inter.put[1][1][0][1] = ff_vvc_put_epel_h4_8_neon;
c->inter.put[1][2][0][1] = ff_vvc_put_epel_h8_8_neon;
c->inter.put[1][3][0][1] = ff_vvc_put_epel_h16_8_neon;
c->inter.put[1][4][0][1] =
c->inter.put[1][5][0][1] =
c->inter.put[1][6][0][1] = ff_vvc_put_epel_h32_8_neon;
c->inter.put[1][1][1][1] = ff_vvc_put_epel_hv4_8_neon;
c->inter.put[1][2][1][1] = ff_vvc_put_epel_hv8_8_neon;
c->inter.put[1][3][1][1] = ff_vvc_put_epel_hv16_8_neon;
c->inter.put[1][4][1][1] = ff_vvc_put_epel_hv32_8_neon;
c->inter.put[1][5][1][1] = ff_vvc_put_epel_hv64_8_neon;
c->inter.put[1][6][1][1] = ff_vvc_put_epel_hv128_8_neon;
c->inter.put_uni[0][1][0][0] = ff_vvc_put_pel_uni_pixels4_8_neon;
c->inter.put_uni[0][2][0][0] = ff_vvc_put_pel_uni_pixels8_8_neon;
c->inter.put_uni[0][3][0][0] = ff_vvc_put_pel_uni_pixels16_8_neon;
c->inter.put_uni[0][4][0][0] = ff_vvc_put_pel_uni_pixels32_8_neon;
c->inter.put_uni[0][5][0][0] = ff_vvc_put_pel_uni_pixels64_8_neon;
c->inter.put_uni[0][6][0][0] = ff_vvc_put_pel_uni_pixels128_8_neon;
c->inter.put_uni[0][1][0][1] = ff_vvc_put_qpel_uni_h4_8_neon;
c->inter.put_uni[0][2][0][1] = ff_vvc_put_qpel_uni_h8_8_neon;
c->inter.put_uni[0][3][0][1] = ff_vvc_put_qpel_uni_h16_8_neon;
c->inter.put_uni[0][4][0][1] =
c->inter.put_uni[0][5][0][1] =
c->inter.put_uni[0][6][0][1] = ff_vvc_put_qpel_uni_h32_8_neon;
c->inter.put_uni_w[0][1][0][0] = ff_vvc_put_pel_uni_w_pixels4_8_neon;
c->inter.put_uni_w[0][2][0][0] = ff_vvc_put_pel_uni_w_pixels8_8_neon;
c->inter.put_uni_w[0][3][0][0] = ff_vvc_put_pel_uni_w_pixels16_8_neon;
c->inter.put_uni_w[0][4][0][0] = ff_vvc_put_pel_uni_w_pixels32_8_neon;
c->inter.put_uni_w[0][5][0][0] = ff_vvc_put_pel_uni_w_pixels64_8_neon;
c->inter.put_uni_w[0][6][0][0] = ff_vvc_put_pel_uni_w_pixels128_8_neon;
c->inter.avg = ff_vvc_avg_8_neon;
for (int i = 0; i < FF_ARRAY_ELEMS(c->sao.band_filter); i++)
c->sao.band_filter[i] = ff_h26x_sao_band_filter_8x8_8_neon;
c->sao.edge_filter[0] = ff_vvc_sao_edge_filter_8x8_8_neon;
for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++)
c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_8_neon;
c->alf.filter[LUMA] = alf_filter_luma_8_neon;
c->alf.filter[CHROMA] = alf_filter_chroma_8_neon;
if (have_i8mm(cpu_flags)) {
c->inter.put[0][1][0][1] = ff_vvc_put_qpel_h4_8_neon_i8mm;
c->inter.put[0][2][0][1] = ff_vvc_put_qpel_h8_8_neon_i8mm;
c->inter.put[0][3][0][1] = ff_vvc_put_qpel_h16_8_neon_i8mm;
c->inter.put[0][4][0][1] = ff_vvc_put_qpel_h32_8_neon_i8mm;
c->inter.put[0][5][0][1] = ff_vvc_put_qpel_h64_8_neon_i8mm;
c->inter.put[0][6][0][1] = ff_vvc_put_qpel_h128_8_neon_i8mm;
c->inter.put[0][1][1][1] = ff_vvc_put_qpel_hv4_8_neon_i8mm;
c->inter.put[0][2][1][1] = ff_vvc_put_qpel_hv8_8_neon_i8mm;
c->inter.put[0][3][1][1] = ff_vvc_put_qpel_hv16_8_neon_i8mm;
c->inter.put[0][4][1][1] = ff_vvc_put_qpel_hv32_8_neon_i8mm;
c->inter.put[0][5][1][1] = ff_vvc_put_qpel_hv64_8_neon_i8mm;
c->inter.put[0][6][1][1] = ff_vvc_put_qpel_hv128_8_neon_i8mm;
c->inter.put[1][1][0][1] = ff_vvc_put_epel_h4_8_neon_i8mm;
c->inter.put[1][2][0][1] = ff_vvc_put_epel_h8_8_neon_i8mm;
c->inter.put[1][3][0][1] = ff_vvc_put_epel_h16_8_neon_i8mm;
c->inter.put[1][4][0][1] = ff_vvc_put_epel_h32_8_neon_i8mm;
c->inter.put[1][5][0][1] = ff_vvc_put_epel_h64_8_neon_i8mm;
c->inter.put[1][6][0][1] = ff_vvc_put_epel_h128_8_neon_i8mm;
c->inter.put[1][1][1][1] = ff_vvc_put_epel_hv4_8_neon_i8mm;
c->inter.put[1][2][1][1] = ff_vvc_put_epel_hv8_8_neon_i8mm;
c->inter.put[1][3][1][1] = ff_vvc_put_epel_hv16_8_neon_i8mm;
c->inter.put[1][4][1][1] = ff_vvc_put_epel_hv32_8_neon_i8mm;
c->inter.put[1][5][1][1] = ff_vvc_put_epel_hv64_8_neon_i8mm;
c->inter.put[1][6][1][1] = ff_vvc_put_epel_hv128_8_neon_i8mm;
}
} else if (bd == 10) {
c->inter.avg = ff_vvc_avg_10_neon;
c->alf.filter[LUMA] = alf_filter_luma_10_neon;
c->alf.filter[CHROMA] = alf_filter_chroma_10_neon;
} else if (bd == 12) {
c->inter.avg = ff_vvc_avg_12_neon;
c->alf.filter[LUMA] = alf_filter_luma_12_neon;
c->alf.filter[CHROMA] = alf_filter_chroma_12_neon;
}
c->inter.sad = ff_vvc_sad_neon;
}
+163
View File
@@ -0,0 +1,163 @@
/*
* Copyright (c) 2024 Zhao Zhili <quinkblack@foxmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#define VVC_MAX_PB_SIZE 128
.macro vvc_avg, bit_depth
.macro vvc_avg_\bit_depth\()_2_4, tap
.if \tap == 2
ldr s0, [src0]
ldr s2, [src1]
.else
ldr d0, [src0]
ldr d2, [src1]
.endif
saddl v4.4s, v0.4h, v2.4h
add v4.4s, v4.4s, v16.4s
sqshrn v4.4h, v4.4s, #(15 - \bit_depth)
.if \bit_depth == 8
sqxtun v4.8b, v4.8h
.if \tap == 2
str h4, [dst]
.else // tap == 4
str s4, [dst]
.endif
.else // bit_depth > 8
smin v4.4h, v4.4h, v17.4h
smax v4.4h, v4.4h, v18.4h
.if \tap == 2
str s4, [dst]
.else
str d4, [dst]
.endif
.endif
add src0, src0, x10
add src1, src1, x10
add dst, dst, dst_stride
.endm
function ff_vvc_avg_\bit_depth\()_neon, export=1
dst .req x0
dst_stride .req x1
src0 .req x2
src1 .req x3
width .req w4
height .req w5
mov x10, #(VVC_MAX_PB_SIZE * 2)
cmp width, #8
.if \bit_depth == 8
movi v16.4s, #64
.else
.if \bit_depth == 10
mov w6, #1023
movi v16.4s, #16
.else
mov w6, #4095
movi v16.4s, #4
.endif
movi v18.8h, #0
dup v17.8h, w6
.endif
b.eq 8f
b.hi 16f
cmp width, #4
b.eq 4f
2: // width == 2
subs height, height, #1
vvc_avg_\bit_depth\()_2_4 2
b.ne 2b
b 32f
4: // width == 4
subs height, height, #1
vvc_avg_\bit_depth\()_2_4 4
b.ne 4b
b 32f
8: // width == 8
ld1 {v0.8h}, [src0], x10
ld1 {v2.8h}, [src1], x10
saddl v4.4s, v0.4h, v2.4h
saddl2 v5.4s, v0.8h, v2.8h
add v4.4s, v4.4s, v16.4s
add v5.4s, v5.4s, v16.4s
sqshrn v4.4h, v4.4s, #(15 - \bit_depth)
sqshrn2 v4.8h, v5.4s, #(15 - \bit_depth)
subs height, height, #1
.if \bit_depth == 8
sqxtun v4.8b, v4.8h
st1 {v4.8b}, [dst], dst_stride
.else
smin v4.8h, v4.8h, v17.8h
smax v4.8h, v4.8h, v18.8h
st1 {v4.8h}, [dst], dst_stride
.endif
b.ne 8b
b 32f
16: // width >= 16
mov w6, width
mov x7, src0
mov x8, src1
mov x9, dst
17:
ldp q0, q1, [x7], #32
ldp q2, q3, [x8], #32
saddl v4.4s, v0.4h, v2.4h
saddl2 v5.4s, v0.8h, v2.8h
saddl v6.4s, v1.4h, v3.4h
saddl2 v7.4s, v1.8h, v3.8h
add v4.4s, v4.4s, v16.4s
add v5.4s, v5.4s, v16.4s
add v6.4s, v6.4s, v16.4s
add v7.4s, v7.4s, v16.4s
sqshrn v4.4h, v4.4s, #(15 - \bit_depth)
sqshrn2 v4.8h, v5.4s, #(15 - \bit_depth)
sqshrn v6.4h, v6.4s, #(15 - \bit_depth)
sqshrn2 v6.8h, v7.4s, #(15 - \bit_depth)
subs w6, w6, #16
.if \bit_depth == 8
sqxtun v4.8b, v4.8h
sqxtun2 v4.16b, v6.8h
str q4, [x9], #16
.else
smin v4.8h, v4.8h, v17.8h
smin v6.8h, v6.8h, v17.8h
smax v4.8h, v4.8h, v18.8h
smax v6.8h, v6.8h, v18.8h
stp q4, q6, [x9], #32
.endif
b.ne 17b
subs height, height, #1
add src0, src0, x10
add src1, src1, x10
add dst, dst, dst_stride
b.ne 16b
32:
ret
endfunc
.endm
vvc_avg 8
vvc_avg 10
vvc_avg 12
+75
View File
@@ -0,0 +1,75 @@
/*
* Copyright (c) 2024 Zhao Zhili <quinkblack@foxmail.com>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#define VVC_MAX_PB_SIZE 128
function ff_vvc_sad_neon, export=1
src0 .req x0
src1 .req x1
dx .req w2
dy .req w3
block_w .req w4
block_h .req w5
sub w7, dx, #4
sub w8, dy, #4
add w6, dx, dy, lsl #7
add w7, w7, w8, lsl #7
sxtw x6, w6
sxtw x7, w7
add src0, src0, x6, lsl #1
sub src1, src1, x7, lsl #1
cmp block_w, #16
movi v16.4s, #0
b.ge 2f
1:
// block_w == 8
ldr q0, [src0]
ldr q2, [src1]
subs block_h, block_h, #2
sabal v16.4s, v0.4h, v2.4h
sabal2 v16.4s, v0.8h, v2.8h
add src0, src0, #(2 * VVC_MAX_PB_SIZE * 2)
add src1, src1, #(2 * VVC_MAX_PB_SIZE * 2)
b.ne 1b
b 4f
2:
// block_w == 16, no block_w > 16 according the spec
movi v17.4s, #0
3:
ldp q0, q1, [src0], #(2 * VVC_MAX_PB_SIZE * 2)
ldp q2, q3, [src1], #(2 * VVC_MAX_PB_SIZE * 2)
subs block_h, block_h, #2
sabal v16.4s, v0.4h, v2.4h
sabal2 v16.4s, v0.8h, v2.8h
sabal v17.4s, v1.4h, v3.4h
sabal2 v17.4s, v1.8h, v3.8h
b.ne 3b
add v16.4s, v16.4s, v17.4s
4:
addv s16, v16.4s
mov w0, v16.s[0]
ret
endfunc