From 68169be97319af5d84ec619635574fcd3e24f113 Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sun, 19 Oct 2025 17:18:27 +0800 Subject: [PATCH] lavf/rkrga: add RKRGA scale, vpp and overlay filters Signed-off-by: nyanmisaka --- configure | 10 + libavfilter/Makefile | 5 + libavfilter/allfilters.c | 3 + libavfilter/rkrga_common.c | 1418 ++++++++++++++++++++++++++++++++ libavfilter/rkrga_common.h | 131 +++ libavfilter/vf_overlay_rkrga.c | 375 +++++++++ libavfilter/vf_vpp_rkrga.c | 581 +++++++++++++ 7 files changed, 2523 insertions(+) create mode 100644 libavfilter/rkrga_common.c create mode 100644 libavfilter/rkrga_common.h create mode 100644 libavfilter/vf_overlay_rkrga.c create mode 100644 libavfilter/vf_vpp_rkrga.c diff --git a/configure b/configure index 92ca15ff8e..c8dced5330 100755 --- a/configure +++ b/configure @@ -354,6 +354,7 @@ External library support: --enable-omx enable OpenMAX IL code [no] --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no] --enable-rkmpp enable Rockchip Media Process Platform code [no] + --enable-rkrga enable Rockchip 2D Raster Graphic Acceleration code [no] --disable-v4l2-m2m disable V4L2 mem2mem code [autodetect] --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect] --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect] @@ -1887,6 +1888,7 @@ EXTERNAL_LIBRARY_VERSION3_LIST=" libvo_amrwbenc mbedtls rkmpp + rkrga " EXTERNAL_LIBRARY_GPLV3_LIST=" @@ -3873,6 +3875,7 @@ overlay_qsv_filter_deps="libmfx" overlay_qsv_filter_select="qsvvpp" overlay_vaapi_filter_deps="vaapi VAProcPipelineCaps_blend_flags" overlay_vulkan_filter_deps="vulkan spirv_compiler" +overlay_rkrga_filter_deps="rkrga" owdenoise_filter_deps="gpl" pad_opencl_filter_deps="opencl" pan_filter_deps="swresample" @@ -3894,6 +3897,7 @@ scale2ref_filter_deps="swscale" scale_filter_deps="swscale" scale_qsv_filter_deps="libmfx" scale_qsv_filter_select="qsvvpp" +scale_rkrga_filter_deps="rkrga" scdet_filter_select="scene_sad" select_filter_select="scene_sad" sharpness_vaapi_filter_deps="vaapi" @@ -3937,6 +3941,7 @@ scale_vt_filter_deps="videotoolbox VTPixelTransferSessionCreate" scale_vulkan_filter_deps="vulkan spirv_compiler" vpp_qsv_filter_deps="libmfx" vpp_qsv_filter_select="qsvvpp" +vpp_rkrga_filter_deps="rkrga" xfade_opencl_filter_deps="opencl" xfade_vulkan_filter_deps="vulkan spirv_compiler" yadif_cuda_filter_deps="ffnvcodec" @@ -7089,6 +7094,11 @@ enabled rkmpp && { require_pkg_config rkmpp rockchip_mpp rockchip/rk { enabled libdrm || die "ERROR: rkmpp requires --enable-libdrm"; } } +enabled rkrga && { require_pkg_config rkrga librga rga/RgaApi.h c_RkRgaBlit && + require_pkg_config rkrga librga rga/im2d.h querystring && + { enabled rkmpp || + die "ERROR: rkrga requires --enable-rkmpp"; } + } enabled vapoursynth && require_pkg_config vapoursynth "vapoursynth-script >= 42" VSScript.h vsscript_init diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 994d9773ba..e2963389c0 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -29,6 +29,7 @@ OBJS-$(HAVE_THREADS) += pthread.o # subsystems OBJS-$(CONFIG_QSVVPP) += qsvvpp.o +OBJS-$(CONFIG_RKRGA) += rkrga_common.o OBJS-$(CONFIG_SCENE_SAD) += scene_sad.o OBJS-$(CONFIG_DNN) += dnn_filter_common.o include $(SRC_PATH)/libavfilter/dnn/Makefile @@ -413,6 +414,7 @@ OBJS-$(CONFIG_OVERLAY_OPENCL_FILTER) += vf_overlay_opencl.o opencl.o \ OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o framesync.o OBJS-$(CONFIG_OVERLAY_VAAPI_FILTER) += vf_overlay_vaapi.o framesync.o vaapi_vpp.o OBJS-$(CONFIG_OVERLAY_VULKAN_FILTER) += vf_overlay_vulkan.o vulkan.o vulkan_filter.o +OBJS-$(CONFIG_OVERLAY_RKRGA_FILTER) += vf_overlay_rkrga.o framesync.o OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o OBJS-$(CONFIG_PAD_OPENCL_FILTER) += vf_pad_opencl.o opencl.o opencl/pad.o @@ -463,6 +465,7 @@ OBJS-$(CONFIG_SCALE_QSV_FILTER) += vf_vpp_qsv.o OBJS-$(CONFIG_SCALE_VAAPI_FILTER) += vf_scale_vaapi.o scale_eval.o vaapi_vpp.o OBJS-$(CONFIG_SCALE_VT_FILTER) += vf_scale_vt.o scale_eval.o OBJS-$(CONFIG_SCALE_VULKAN_FILTER) += vf_scale_vulkan.o vulkan.o vulkan_filter.o +OBJS-$(CONFIG_SCALE_RKRGA_FILTER) += vf_vpp_rkrga.o scale_eval.o OBJS-$(CONFIG_SCALE2REF_FILTER) += vf_scale.o scale_eval.o OBJS-$(CONFIG_SCALE2REF_NPP_FILTER) += vf_scale_npp.o scale_eval.o OBJS-$(CONFIG_SCDET_FILTER) += vf_scdet.o @@ -553,6 +556,7 @@ OBJS-$(CONFIG_VIF_FILTER) += vf_vif.o framesync.o OBJS-$(CONFIG_VIGNETTE_FILTER) += vf_vignette.o OBJS-$(CONFIG_VMAFMOTION_FILTER) += vf_vmafmotion.o framesync.o OBJS-$(CONFIG_VPP_QSV_FILTER) += vf_vpp_qsv.o +OBJS-$(CONFIG_VPP_RKRGA_FILTER) += vf_vpp_rkrga.o scale_eval.o OBJS-$(CONFIG_VSTACK_FILTER) += vf_stack.o framesync.o OBJS-$(CONFIG_W3FDIF_FILTER) += vf_w3fdif.o OBJS-$(CONFIG_WAVEFORM_FILTER) += vf_waveform.o @@ -653,6 +657,7 @@ SKIPHEADERS-$(CONFIG_LCMS2) += fflcms2.h SKIPHEADERS-$(CONFIG_LIBVIDSTAB) += vidstabutils.h SKIPHEADERS-$(CONFIG_QSVVPP) += qsvvpp.h stack_internal.h +SKIPHEADERS-$(CONFIG_RKRGA) += rkrga_common.h SKIPHEADERS-$(CONFIG_OPENCL) += opencl.h SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_vpp.h stack_internal.h SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_filter.h diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 149bf50997..79f5534974 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -389,6 +389,7 @@ extern const AVFilter ff_vf_overlay_qsv; extern const AVFilter ff_vf_overlay_vaapi; extern const AVFilter ff_vf_overlay_vulkan; extern const AVFilter ff_vf_overlay_cuda; +extern const AVFilter ff_vf_overlay_rkrga; extern const AVFilter ff_vf_owdenoise; extern const AVFilter ff_vf_pad; extern const AVFilter ff_vf_pad_opencl; @@ -436,6 +437,7 @@ extern const AVFilter ff_vf_scale_qsv; extern const AVFilter ff_vf_scale_vaapi; extern const AVFilter ff_vf_scale_vt; extern const AVFilter ff_vf_scale_vulkan; +extern const AVFilter ff_vf_scale_rkrga; extern const AVFilter ff_vf_scale2ref; extern const AVFilter ff_vf_scale2ref_npp; extern const AVFilter ff_vf_scdet; @@ -522,6 +524,7 @@ extern const AVFilter ff_vf_vif; extern const AVFilter ff_vf_vignette; extern const AVFilter ff_vf_vmafmotion; extern const AVFilter ff_vf_vpp_qsv; +extern const AVFilter ff_vf_vpp_rkrga; extern const AVFilter ff_vf_vstack; extern const AVFilter ff_vf_w3fdif; extern const AVFilter ff_vf_waveform; diff --git a/libavfilter/rkrga_common.c b/libavfilter/rkrga_common.c new file mode 100644 index 0000000000..f308fd14ea --- /dev/null +++ b/libavfilter/rkrga_common.c @@ -0,0 +1,1418 @@ +/* + * Copyright (c) 2023 NyanMisaka + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Rockchip RGA (2D Raster Graphic Acceleration) base function + */ + +#include "libavutil/common.h" +#include "libavutil/pixdesc.h" + +#include "internal.h" +#include "video.h" + +#include "rkrga_common.h" + +typedef struct RGAAsyncFrame { + RGAFrame *src; + RGAFrame *dst; + RGAFrame *pat; +} RGAAsyncFrame; + +typedef struct RGAFormatMap { + enum AVPixelFormat pix_fmt; + enum _Rga_SURF_FORMAT rga_fmt; +} RGAFormatMap; + +#define RK_FORMAT_YCbCr_444_SP (0x32 << 8) +#define RK_FORMAT_YCrCb_444_SP (0x33 << 8) + +#define YUV_FORMATS \ + { AV_PIX_FMT_GRAY8, RK_FORMAT_YCbCr_400 }, /* RGA2 only */ \ + { AV_PIX_FMT_YUV420P, RK_FORMAT_YCbCr_420_P }, /* RGA2 only */ \ + { AV_PIX_FMT_YUVJ420P, RK_FORMAT_YCbCr_420_P }, /* RGA2 only */ \ + { AV_PIX_FMT_YUV422P, RK_FORMAT_YCbCr_422_P }, /* RGA2 only */ \ + { AV_PIX_FMT_YUVJ422P, RK_FORMAT_YCbCr_422_P }, /* RGA2 only */ \ + { AV_PIX_FMT_NV12, RK_FORMAT_YCbCr_420_SP }, \ + { AV_PIX_FMT_NV21, RK_FORMAT_YCrCb_420_SP }, \ + { AV_PIX_FMT_NV16, RK_FORMAT_YCbCr_422_SP }, \ + { AV_PIX_FMT_NV24, RK_FORMAT_YCbCr_444_SP }, /* RGA2-Pro only */ \ + { AV_PIX_FMT_NV42, RK_FORMAT_YCrCb_444_SP }, /* RGA2-Pro only */ \ + { AV_PIX_FMT_P010, RK_FORMAT_YCbCr_420_SP_10B }, /* RGA3 only */ \ + { AV_PIX_FMT_P210, RK_FORMAT_YCbCr_422_SP_10B }, /* RGA3 only */ \ + { AV_PIX_FMT_NV15, RK_FORMAT_YCbCr_420_SP_10B }, /* RGA2 only input, aka P010 compact */ \ + { AV_PIX_FMT_NV20, RK_FORMAT_YCbCr_422_SP_10B }, /* RGA2 only input, aka P210 compact */ \ + { AV_PIX_FMT_YUYV422, RK_FORMAT_YUYV_422 }, \ + { AV_PIX_FMT_YVYU422, RK_FORMAT_YVYU_422 }, \ + { AV_PIX_FMT_UYVY422, RK_FORMAT_UYVY_422 }, + +#define RGB_FORMATS \ + { AV_PIX_FMT_RGB555LE, RK_FORMAT_BGRA_5551 }, /* RGA2 only */ \ + { AV_PIX_FMT_BGR555LE, RK_FORMAT_RGBA_5551 }, /* RGA2 only */ \ + { AV_PIX_FMT_RGB565LE, RK_FORMAT_BGR_565 }, \ + { AV_PIX_FMT_BGR565LE, RK_FORMAT_RGB_565 }, \ + { AV_PIX_FMT_RGB24, RK_FORMAT_RGB_888 }, \ + { AV_PIX_FMT_BGR24, RK_FORMAT_BGR_888 }, \ + { AV_PIX_FMT_RGBA, RK_FORMAT_RGBA_8888 }, \ + { AV_PIX_FMT_RGB0, RK_FORMAT_RGBA_8888 }, /* RK_FORMAT_RGBX_8888 triggers RGA2 on multicore RGA */ \ + { AV_PIX_FMT_BGRA, RK_FORMAT_BGRA_8888 }, \ + { AV_PIX_FMT_BGR0, RK_FORMAT_BGRA_8888 }, /* RK_FORMAT_BGRX_8888 triggers RGA2 on multicore RGA */ \ + { AV_PIX_FMT_ARGB, RK_FORMAT_ARGB_8888 }, /* RGA3 only input */ \ + { AV_PIX_FMT_0RGB, RK_FORMAT_ARGB_8888 }, /* RGA3 only input */ \ + { AV_PIX_FMT_ABGR, RK_FORMAT_ABGR_8888 }, /* RGA3 only input */ \ + { AV_PIX_FMT_0BGR, RK_FORMAT_ABGR_8888 }, /* RGA3 only input */ + +static const RGAFormatMap supported_formats_main[] = { + YUV_FORMATS + RGB_FORMATS +}; + +static const RGAFormatMap supported_formats_overlay[] = { + RGB_FORMATS +}; +#undef YUV_FORMATS +#undef RGB_FORMATS + +static int map_av_to_rga_format(enum AVPixelFormat in_format, + enum _Rga_SURF_FORMAT *out_format, int is_overlay) +{ + int i; + + if (is_overlay) + goto overlay; + + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats_main); i++) { + if (supported_formats_main[i].pix_fmt == in_format) { + if (out_format) + *out_format = supported_formats_main[i].rga_fmt; + return 1; + } + } + return 0; + +overlay: + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats_overlay); i++) { + if (supported_formats_overlay[i].pix_fmt == in_format) { + if (out_format) + *out_format = supported_formats_overlay[i].rga_fmt; + return 1; + } + } + return 0; +} + +static int get_pixel_stride(const AVDRMObjectDescriptor *object, + const AVDRMLayerDescriptor *layer, + int is_rgb, int is_planar, + float bytes_pp, int *ws, int *hs) +{ + const AVDRMPlaneDescriptor *plane0, *plane1; + const int is_packed_fmt = is_rgb || (!is_rgb && !is_planar); + + if (!object || !layer || !ws || !hs || bytes_pp <= 0) + return AVERROR(EINVAL); + + plane0 = &layer->planes[0]; + plane1 = &layer->planes[1]; + + *ws = is_packed_fmt ? + (plane0->pitch / bytes_pp) : + plane0->pitch; + *hs = is_packed_fmt ? + ALIGN_DOWN(object->size / plane0->pitch, is_rgb ? 1 : 2) : + (plane1->offset / plane0->pitch); + + return (*ws > 0 && *hs > 0) ? 0 : AVERROR(EINVAL); +} + +static int get_afbc_pixel_stride(float bytes_pp, int *stride, int reverse) +{ + if (!stride || *stride <= 0 || bytes_pp <= 0) + return AVERROR(EINVAL); + + *stride = reverse ? (*stride / bytes_pp) : (*stride * bytes_pp); + + return (*stride > 0) ? 0 : AVERROR(EINVAL); +} + +/* Canonical formats: https://dri.freedesktop.org/docs/drm/gpu/afbc.html */ +static uint32_t get_drm_afbc_format(enum AVPixelFormat pix_fmt) +{ + switch (pix_fmt) { + case AV_PIX_FMT_NV12: return DRM_FORMAT_YUV420_8BIT; + case AV_PIX_FMT_NV15: return DRM_FORMAT_YUV420_10BIT; + case AV_PIX_FMT_NV16: return DRM_FORMAT_YUYV; + case AV_PIX_FMT_NV20: return DRM_FORMAT_Y210; + case AV_PIX_FMT_NV24: return DRM_FORMAT_VUY888; + case AV_PIX_FMT_RGB565LE: return DRM_FORMAT_RGB565; + case AV_PIX_FMT_BGR565LE: return DRM_FORMAT_BGR565; + case AV_PIX_FMT_RGB24: return DRM_FORMAT_RGB888; + case AV_PIX_FMT_BGR24: return DRM_FORMAT_BGR888; + case AV_PIX_FMT_RGBA: return DRM_FORMAT_ABGR8888; + case AV_PIX_FMT_RGB0: return DRM_FORMAT_XBGR8888; + case AV_PIX_FMT_BGRA: return DRM_FORMAT_ARGB8888; + case AV_PIX_FMT_BGR0: return DRM_FORMAT_XRGB8888; + default: return DRM_FORMAT_INVALID; + } +} + +static uint32_t get_drm_rfbc_format(enum AVPixelFormat pix_fmt) +{ + switch (pix_fmt) { + case AV_PIX_FMT_NV12: return DRM_FORMAT_YUV420_8BIT; + case AV_PIX_FMT_NV15: return DRM_FORMAT_YUV420_10BIT; + case AV_PIX_FMT_NV16: return DRM_FORMAT_YUYV; + case AV_PIX_FMT_NV20: return DRM_FORMAT_Y210; + case AV_PIX_FMT_NV24: return DRM_FORMAT_VUY888; + default: return DRM_FORMAT_INVALID; + } +} + +static int is_pixel_stride_rga3_compat(int ws, int hs, + enum _Rga_SURF_FORMAT fmt) +{ + switch (fmt) { + case RK_FORMAT_YCbCr_420_SP: + case RK_FORMAT_YCrCb_420_SP: + case RK_FORMAT_YCbCr_422_SP: return !(ws % 16) && !(hs % 2); + case RK_FORMAT_YCbCr_420_SP_10B: + case RK_FORMAT_YCbCr_422_SP_10B: return !(ws % 64) && !(hs % 2); + case RK_FORMAT_YUYV_422: + case RK_FORMAT_YVYU_422: + case RK_FORMAT_UYVY_422: return !(ws % 8) && !(hs % 2); + case RK_FORMAT_RGB_565: + case RK_FORMAT_BGR_565: return !(ws % 8); + case RK_FORMAT_RGB_888: + case RK_FORMAT_BGR_888: return !(ws % 16); + case RK_FORMAT_RGBA_8888: + case RK_FORMAT_BGRA_8888: + case RK_FORMAT_ARGB_8888: + case RK_FORMAT_ABGR_8888: return !(ws % 4); + default: return 0; + } +} + +static void clear_unused_frames(RGAFrame *list) +{ + while (list) { + if (list->queued == 1 && !list->locked) { + av_frame_free(&list->frame); + list->queued = 0; + } + list = list->next; + } +} + +static void clear_frame_list(RGAFrame **list) +{ + while (*list) { + RGAFrame *frame = NULL; + + frame = *list; + *list = (*list)->next; + av_frame_free(&frame->frame); + av_freep(&frame); + } +} + +static RGAFrame *get_free_frame(RGAFrame **list) +{ + RGAFrame *out = *list; + + for (; out; out = out->next) { + if (!out->queued) { + out->queued = 1; + break; + } + } + + if (!out) { + out = av_mallocz(sizeof(*out)); + if (!out) { + av_log(NULL, AV_LOG_ERROR, "Cannot alloc new output frame\n"); + return NULL; + } + out->queued = 1; + out->next = *list; + *list = out; + } + + return out; +} + +static void set_colorspace_info(RGAFrameInfo *in_info, + enum AVColorSpace in_spc, + enum AVColorRange in_rng, + RGAFrameInfo *out_info, + enum AVColorSpace *out_spc, + enum AVColorRange *out_rng, + int *color_space_mode, + int is_rga2_used) +{ + int rgb_in, rgb_out, out_mode = 0; + + if (!in_info || !out_info || !color_space_mode) + return; + + rgb_in = in_info->pix_desc->flags & AV_PIX_FMT_FLAG_RGB; + rgb_out = out_info->pix_desc->flags & AV_PIX_FMT_FLAG_RGB; + + /* rgb2yuv */ + if (rgb_in && !rgb_out) { + /* rgb full -> yuv full/limit */ + if (in_rng == AVCOL_RANGE_JPEG) { + if ((out_rng && *out_rng == AVCOL_RANGE_JPEG) || + (out_spc && *out_spc == AVCOL_SPC_BT470BG)) { + if (out_spc) + *out_spc = AVCOL_SPC_BT470BG; + + if (out_rng && *out_rng == AVCOL_RANGE_JPEG) + out_mode = 1 << 2; /* IM_RGB_TO_YUV_BT601_FULL */ + else { + if (out_rng) + *out_rng = AVCOL_RANGE_MPEG; + out_mode = 2 << 2; /* IM_RGB_TO_YUV_BT601_LIMIT */ + } + } else { + if (out_spc) + *out_spc = AVCOL_SPC_BT709; + if (out_rng) + *out_rng = AVCOL_RANGE_MPEG; + out_mode = is_rga2_used ? (0xb << 8) /* rgb2yuv_709_limit */ + : (3 << 2); /* IM_RGB_TO_YUV_BT709_LIMIT */ + } + } + if (out_mode) + *color_space_mode |= out_mode; + } + /* yuv2rgb */ + else if (!rgb_in && rgb_out) { + /* yuv full/limit -> rgb full */ + switch (in_rng) { + case AVCOL_RANGE_MPEG: + if (in_spc == AVCOL_SPC_BT709) + out_mode = 3 << 0; /* IM_YUV_TO_RGB_BT709_LIMIT */ + if (in_spc == AVCOL_SPC_BT470BG) + out_mode = 1 << 0; /* IM_YUV_TO_RGB_BT601_LIMIT */ + break; + case AVCOL_RANGE_JPEG: +#if 0 + if (in_spc == AVCOL_SPC_BT709) + out_mode = 0xc << 8; /* yuv2rgb_709_full */ + if (in_spc == AVCOL_SPC_BT470BG) +#endif + out_mode = 2 << 0; /* IM_YUV_TO_RGB_BT601_FULL */ + break; + } + if (out_spc) + *out_spc = AVCOL_SPC_RGB; + if (out_rng) + *out_rng = AVCOL_RANGE_JPEG; + if (out_mode) + *color_space_mode |= out_mode; + } + /* passthrough */ + else { + if (out_spc) + *out_spc = in_spc; + if (out_rng) + *out_rng = in_rng; + } + + /* yuvj2yuv */ + if ((in_info->pix_fmt == AV_PIX_FMT_YUVJ420P || + in_info->pix_fmt == AV_PIX_FMT_YUVJ422P) && !rgb_out) { + if (out_rng) + *out_rng = AVCOL_RANGE_JPEG; + } +} + +static int verify_rga_frame_info_io_dynamic(AVFilterContext *avctx, + RGAFrameInfo *in, RGAFrameInfo *out) +{ + RKRGAContext *r = avctx->priv; + + if (!in || !out) + return AVERROR(EINVAL); + + if (r->is_rga2_used && !r->has_rga2) { + av_log(avctx, AV_LOG_ERROR, "RGA2 is requested but not available\n"); + return AVERROR(ENOSYS); + } + if (r->is_rga2_used && + (in->pix_fmt == AV_PIX_FMT_P010 || + out->pix_fmt == AV_PIX_FMT_P010)) { + av_log(avctx, AV_LOG_ERROR, "'%s' is not supported if RGA2 is requested\n", + av_get_pix_fmt_name(AV_PIX_FMT_P010)); + return AVERROR(ENOSYS); + } + if (r->is_rga2_used && + (in->pix_fmt == AV_PIX_FMT_P210 || + out->pix_fmt == AV_PIX_FMT_P210)) { + av_log(avctx, AV_LOG_ERROR, "'%s' is not supported if RGA2 is requested\n", + av_get_pix_fmt_name(AV_PIX_FMT_P210)); + return AVERROR(ENOSYS); + } + if (r->is_rga2_used && + (out->pix_fmt == AV_PIX_FMT_NV15 || + out->pix_fmt == AV_PIX_FMT_NV20)) { + av_log(avctx, AV_LOG_ERROR, "'%s' as output is not supported if RGA2 is requested\n", + av_get_pix_fmt_name(out->pix_fmt)); + return AVERROR(ENOSYS); + } + if (!r->has_rga2p && r->is_rga2_used && in->crop && in->pix_desc->comp[0].depth >= 10) { + av_log(avctx, AV_LOG_ERROR, "Cropping 10-bit '%s' input is not supported if RGA2 (non-Pro) is requested\n", + av_get_pix_fmt_name(in->pix_fmt)); + return AVERROR(ENOSYS); + } + if (r->is_rga2_used && !r->has_rga2p && + (out->act_w > 4096 || out->act_h > 4096)) { + av_log(avctx, AV_LOG_ERROR, "Max supported output size of RGA2 (non-Pro) is 4096x4096\n"); + return AVERROR(EINVAL); + } + if (!r->is_rga2_used && + (in->act_w < 68 || in->act_h < 2)) { + av_log(avctx, AV_LOG_ERROR, "Min supported input size of RGA3 is 68x2\n"); + return AVERROR(EINVAL); + } + if (!r->is_rga2_used && + (out->act_w > 8128 || out->act_h > 8128)) { + av_log(avctx, AV_LOG_ERROR, "Max supported output size of RGA3 is 8128x8128\n"); + return AVERROR(EINVAL); + } + + return 0; +} + +static RGAFrame *submit_frame(RKRGAContext *r, AVFilterLink *inlink, + const AVFrame *picref, int do_overlay, int pat_preproc) +{ + RGAFrame *rga_frame; + AVFilterContext *ctx = inlink->dst; + rga_info_t info = { .mmuFlag = 1, }; + int nb_link = FF_INLINK_IDX(inlink); + RGAFrameInfo *in_info = &r->in_rga_frame_infos[nb_link]; + RGAFrameInfo *out_info = &r->out_rga_frame_info; + int w_stride = 0, h_stride = 0; + const AVDRMFrameDescriptor *desc; + const AVDRMLayerDescriptor *layer; + const AVDRMPlaneDescriptor *plane0; + RGAFrame **frame_list = NULL; + int is_afbc = 0, is_rfbc = 0; + int ret, is_fbc = 0; + + if (pat_preproc && !nb_link) + return NULL; + + frame_list = nb_link ? + (pat_preproc ? &r->pat_preproc_frame_list : &r->pat_frame_list) : &r->src_frame_list; + + clear_unused_frames(*frame_list); + + rga_frame = get_free_frame(frame_list); + if (!rga_frame) + return NULL; + + if (picref->format != AV_PIX_FMT_DRM_PRIME) { + av_log(ctx, AV_LOG_ERROR, "RGA gets a wrong frame\n"); + return NULL; + } + rga_frame->frame = av_frame_clone(picref); + + desc = (AVDRMFrameDescriptor *)rga_frame->frame->data[0]; + if (desc->objects[0].fd < 0) + return NULL; + + is_afbc = drm_is_afbc(desc->objects[0].format_modifier); + is_rfbc = drm_is_rfbc(desc->objects[0].format_modifier); + is_fbc = is_afbc || is_rfbc; + if (!is_fbc) { + ret = get_pixel_stride(&desc->objects[0], + &desc->layers[0], + (in_info->pix_desc->flags & AV_PIX_FMT_FLAG_RGB), + (in_info->pix_desc->flags & AV_PIX_FMT_FLAG_PLANAR), + in_info->bytes_pp, &w_stride, &h_stride); + if (ret < 0 || !w_stride || !h_stride) { + av_log(ctx, AV_LOG_ERROR, "Failed to get frame strides\n"); + return NULL; + } + } + + info.fd = desc->objects[0].fd; + info.format = in_info->rga_fmt; + info.in_fence_fd = -1; + info.out_fence_fd = -1; + + if (in_info->uncompact_10b_msb) + info.is_10b_compact = info.is_10b_endian = 1; + + if (!nb_link) { + info.rotation = in_info->rotate_mode; + info.blend = (do_overlay && !pat_preproc) ? in_info->blend_mode : 0; + } + + if (is_fbc && !r->has_rga2p && (r->is_rga2_used || out_info->scheduler_core == 0x4)) { + av_log(ctx, AV_LOG_ERROR, "Input format '%s' with AFBC modifier is not supported by RGA2 (non-Pro)\n", + av_get_pix_fmt_name(in_info->pix_fmt)); + return NULL; + } + + /* verify inputs pixel stride */ + if (out_info->scheduler_core > 0 && + out_info->scheduler_core == (out_info->scheduler_core & 0x3)) { + if (!is_afbc && !is_pixel_stride_rga3_compat(w_stride, h_stride, in_info->rga_fmt)) { + r->is_rga2_used = 1; + av_log(ctx, AV_LOG_WARNING, "Input pixel stride (%dx%d) format '%s' is not supported by RGA3\n", + w_stride, h_stride, av_get_pix_fmt_name(in_info->pix_fmt)); + } + + if ((ret = verify_rga_frame_info_io_dynamic(ctx, in_info, out_info)) < 0) + return NULL; + + if (r->is_rga2_used) + out_info->scheduler_core = 0x4; + } + + if (pat_preproc) { + RGAFrameInfo *in0_info = &r->in_rga_frame_infos[0]; + rga_set_rect(&info.rect, 0, 0, + FFMIN((in0_info->act_w - in_info->overlay_x), in_info->act_w), + FFMIN((in0_info->act_h - in_info->overlay_y), in_info->act_h), + w_stride, h_stride, in_info->rga_fmt); + } else + rga_set_rect(&info.rect, in_info->act_x, in_info->act_y, + in_info->act_w, in_info->act_h, + w_stride, h_stride, in_info->rga_fmt); + + if (is_fbc) { + int afbc_offset_y = 0; + int fbc_align_w = + is_afbc ? RK_RGA_AFBC_16x16_STRIDE_ALIGN : RK_RGA_RFBC_64x4_STRIDE_ALIGN_W; + int fbc_align_h = + is_afbc ? RK_RGA_AFBC_16x16_STRIDE_ALIGN : RK_RGA_RFBC_64x4_STRIDE_ALIGN_H; + uint32_t drm_fbc_fmt = + is_afbc ? get_drm_afbc_format(in_info->pix_fmt) : get_drm_rfbc_format(in_info->pix_fmt); + + if (rga_frame->frame->crop_top > 0) { + afbc_offset_y = is_afbc ? rga_frame->frame->crop_top : 0; + info.rect.yoffset += afbc_offset_y; + } + + layer = &desc->layers[0]; + plane0 = &layer->planes[0]; + if (drm_fbc_fmt == layer->format) { + info.rect.wstride = plane0->pitch; + if ((ret = get_afbc_pixel_stride(in_info->bytes_pp, &info.rect.wstride, 1)) < 0) + return NULL; + + if (info.rect.wstride % fbc_align_w) + info.rect.wstride = FFALIGN(inlink->w, fbc_align_w); + + info.rect.hstride = FFALIGN(inlink->h + afbc_offset_y, fbc_align_h); + } else { + av_log(ctx, AV_LOG_ERROR, "Input format '%s' with AFBC/RFBC modifier is not supported\n", + av_get_pix_fmt_name(in_info->pix_fmt)); + return NULL; + } + + info.rd_mode = + is_afbc ? (1 << 1) /* IM_AFBC16x16_MODE */ + : (1 << 4); /* IM_RKFBC64x4_MODE */ + } + + rga_frame->info = info; + + return rga_frame; +} + +static RGAFrame *query_frame(RKRGAContext *r, AVFilterLink *outlink, + const AVFrame *in, const AVFrame *picref_pat, int pat_preproc) +{ + AVFilterContext *ctx = outlink->src; + AVFilterLink *inlink = ctx->inputs[0]; + RGAFrame *out_frame; + rga_info_t info = { .mmuFlag = 1, }; + RGAFrameInfo *in0_info = &r->in_rga_frame_infos[0]; + RGAFrameInfo *in1_info = ctx->nb_inputs > 1 ? &r->in_rga_frame_infos[1] : NULL; + RGAFrameInfo *out_info = pat_preproc ? in1_info : &r->out_rga_frame_info; + AVBufferRef *hw_frame_ctx = pat_preproc ? r->pat_preproc_hwframes_ctx : outlink->hw_frames_ctx; + int w_stride = 0, h_stride = 0; + AVDRMFrameDescriptor *desc; + AVDRMLayerDescriptor *layer; + RGAFrame **frame_list = NULL; + int ret, is_afbc = 0; + + if (!out_info || !hw_frame_ctx) + return NULL; + + frame_list = pat_preproc ? &r->pat_frame_list : &r->dst_frame_list; + + clear_unused_frames(*frame_list); + + out_frame = get_free_frame(frame_list); + if (!out_frame) + return NULL; + + out_frame->frame = av_frame_alloc(); + if (!out_frame->frame) + return NULL; + + if (in && (ret = av_frame_copy_props(out_frame->frame, in)) < 0) { + av_log(ctx, AV_LOG_ERROR, "Failed to copy metadata fields from in to out: %d\n", ret); + goto fail; + } + out_frame->frame->crop_top = 0; + + if ((ret = av_hwframe_get_buffer(hw_frame_ctx, out_frame->frame, 0)) < 0) { + av_log(ctx, AV_LOG_ERROR, "Cannot allocate an internal frame: %d\n", ret); + goto fail; + } + + desc = (AVDRMFrameDescriptor *)out_frame->frame->data[0]; + if (desc->objects[0].fd < 0) + goto fail; + + if (r->is_rga2_used || out_info->scheduler_core == 0x4) { + if (!r->has_rga2p && pat_preproc && (info.rect.width > 4096 || info.rect.height > 4096)) { + av_log(ctx, AV_LOG_ERROR, "Max supported output size of RGA2 (non-Pro) is 4096x4096\n"); + goto fail; + } + if (r->afbc_out && !pat_preproc) { + av_log(ctx, AV_LOG_WARNING, "Output format '%s' with AFBC modifier is not supported by RGA2\n", + av_get_pix_fmt_name(out_info->pix_fmt)); + r->afbc_out = 0; + } + } + + is_afbc = r->afbc_out && !pat_preproc; + ret = get_pixel_stride(&desc->objects[0], + &desc->layers[0], + (out_info->pix_desc->flags & AV_PIX_FMT_FLAG_RGB), + (out_info->pix_desc->flags & AV_PIX_FMT_FLAG_PLANAR), + out_info->bytes_pp, &w_stride, &h_stride); + if (!is_afbc && (ret < 0 || !w_stride || !h_stride)) { + av_log(ctx, AV_LOG_ERROR, "Failed to get frame strides\n"); + goto fail; + } + + info.fd = desc->objects[0].fd; + info.format = out_info->rga_fmt; + info.core = out_info->scheduler_core; + info.in_fence_fd = -1; + info.out_fence_fd = -1; + info.sync_mode = RGA_BLIT_ASYNC; + + if (out_info->uncompact_10b_msb) + info.is_10b_compact = info.is_10b_endian = 1; + + if (!pat_preproc) { + int is_rga2_used = r->is_rga2_used || out_info->scheduler_core == (out_info->scheduler_core & 0xc); + +#ifdef RGA_NORMAL_DST_FULL_CSC_FIXUP + if (in1_info && picref_pat) { + enum AVColorSpace pat_colorspace = picref_pat->colorspace; + enum AVColorRange pat_color_range = picref_pat->color_range; + /* yuv2rgb src->pat */ + set_colorspace_info(in0_info, in->colorspace, in->color_range, + in1_info, &pat_colorspace, &pat_color_range, + &info.color_space_mode, is_rga2_used); + /* rgb2yuv pat->dst */ + set_colorspace_info(in1_info, pat_colorspace, pat_color_range, + out_info, &out_frame->frame->colorspace, &out_frame->frame->color_range, + &info.color_space_mode, is_rga2_used); + } else +#endif + { + set_colorspace_info(in0_info, in->colorspace, in->color_range, + out_info, &out_frame->frame->colorspace, &out_frame->frame->color_range, + &info.color_space_mode, is_rga2_used); + } + } + + if (pat_preproc) + rga_set_rect(&info.rect, in1_info->overlay_x, in1_info->overlay_y, + FFMIN((in0_info->act_w - in1_info->overlay_x), in1_info->act_w), + FFMIN((in0_info->act_h - in1_info->overlay_y), in1_info->act_h), + w_stride, h_stride, in1_info->rga_fmt); + else + rga_set_rect(&info.rect, out_info->act_x, out_info->act_y, + out_info->act_w, out_info->act_h, + w_stride, h_stride, out_info->rga_fmt); + + if (is_afbc) { + uint32_t drm_afbc_fmt = get_drm_afbc_format(out_info->pix_fmt); + + if (drm_afbc_fmt == DRM_FORMAT_INVALID) { + av_log(ctx, AV_LOG_WARNING, "Output format '%s' with AFBC modifier is not supported\n", + av_get_pix_fmt_name(out_info->pix_fmt)); + r->afbc_out = 0; + goto exit; + } + + w_stride = FFALIGN(pat_preproc ? inlink->w : outlink->w, RK_RGA_AFBC_16x16_STRIDE_ALIGN); + h_stride = FFALIGN(pat_preproc ? inlink->h : outlink->h, RK_RGA_AFBC_16x16_STRIDE_ALIGN); + + if ((info.rect.format == RK_FORMAT_YCbCr_420_SP_10B || + info.rect.format == RK_FORMAT_YCbCr_422_SP_10B) && (w_stride % 64)) { + av_log(ctx, AV_LOG_WARNING, "Output pixel wstride '%d' format '%s' is not supported by RGA3 AFBC\n", + w_stride, av_get_pix_fmt_name(out_info->pix_fmt)); + r->afbc_out = 0; + goto exit; + } + +#ifndef RGA_NORMAL_FBCE_RGB_BGR_FIXUP + /* Inverted RGB/BGR order in FBCE */ + switch (info.rect.format) { + case RK_FORMAT_RGBA_8888: + info.rect.format = RK_FORMAT_BGRA_8888; + break; + case RK_FORMAT_BGRA_8888: + info.rect.format = RK_FORMAT_RGBA_8888; + break; + } +#endif + + info.rect.wstride = w_stride; + info.rect.hstride = h_stride; + info.rd_mode = 1 << 1; /* IM_AFBC16x16_MODE */ + + desc->objects[0].format_modifier = + DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_BLOCK_SIZE_16x16); + + layer = &desc->layers[0]; + layer->format = drm_afbc_fmt; + layer->nb_planes = 1; + + layer->planes[0].offset = 0; + layer->planes[0].pitch = info.rect.wstride; + + if ((ret = get_afbc_pixel_stride(out_info->bytes_pp, (int *)&layer->planes[0].pitch, 0)) < 0) + goto fail; + } + +exit: + out_frame->info = info; + + return out_frame; + +fail: + if (out_frame && out_frame->frame) + av_frame_free(&out_frame->frame); + + return NULL; +} + +static av_cold int init_hwframes_ctx(AVFilterContext *avctx) +{ + RKRGAContext *r = avctx->priv; + AVFilterLink *inlink = avctx->inputs[0]; + AVFilterLink *outlink = avctx->outputs[0]; + AVHWFramesContext *hwfc_in; + AVHWFramesContext *hwfc_out; + AVBufferRef *hwfc_out_ref; + AVHWDeviceContext *device_ctx; + AVBufferRef *device_ref; + AVRKMPPFramesContext *rkmpp_fc; + int ret; + + if (!inlink->hw_frames_ctx) + return AVERROR(EINVAL); + + hwfc_in = (AVHWFramesContext *)inlink->hw_frames_ctx->data; + device_ref = hwfc_in->device_ref; + device_ctx = (AVHWDeviceContext *)device_ref->data; + + if (!device_ctx || device_ctx->type != AV_HWDEVICE_TYPE_RKMPP) { + if (avctx->hw_device_ctx) { + device_ref = avctx->hw_device_ctx; + device_ctx = (AVHWDeviceContext *)device_ref->data; + } + if (!device_ctx || device_ctx->type != AV_HWDEVICE_TYPE_RKMPP) { + av_log(avctx, AV_LOG_ERROR, "No RKMPP hardware context provided\n"); + return AVERROR(EINVAL); + } + } + + hwfc_out_ref = av_hwframe_ctx_alloc(device_ref); + if (!hwfc_out_ref) + return AVERROR(ENOMEM); + + hwfc_out = (AVHWFramesContext *)hwfc_out_ref->data; + hwfc_out->format = AV_PIX_FMT_DRM_PRIME; + hwfc_out->sw_format = r->out_sw_format; + hwfc_out->width = outlink->w; + hwfc_out->height = outlink->h; + + rkmpp_fc = hwfc_out->hwctx; + rkmpp_fc->flags |= MPP_BUFFER_FLAGS_CACHABLE; + + ret = av_hwframe_ctx_init(hwfc_out_ref); + if (ret < 0) { + av_buffer_unref(&hwfc_out_ref); + av_log(avctx, AV_LOG_ERROR, "Error creating frames_ctx for output pad: %d\n", ret); + return ret; + } + + av_buffer_unref(&outlink->hw_frames_ctx); + outlink->hw_frames_ctx = hwfc_out_ref; + + return 0; +} + +static av_cold int init_pat_preproc_hwframes_ctx(AVFilterContext *avctx) +{ + RKRGAContext *r = avctx->priv; + AVFilterLink *inlink0 = avctx->inputs[0]; + AVFilterLink *inlink1 = avctx->inputs[1]; + AVHWFramesContext *hwfc_in0, *hwfc_in1; + AVHWFramesContext *hwfc_pat; + AVBufferRef *hwfc_pat_ref; + AVHWDeviceContext *device_ctx0; + AVBufferRef *device_ref0; + int ret; + + if (!inlink0->hw_frames_ctx || !inlink1->hw_frames_ctx) + return AVERROR(EINVAL); + + hwfc_in0 = (AVHWFramesContext *)inlink0->hw_frames_ctx->data; + hwfc_in1 = (AVHWFramesContext *)inlink1->hw_frames_ctx->data; + device_ref0 = hwfc_in0->device_ref; + device_ctx0 = (AVHWDeviceContext *)device_ref0->data; + + if (!device_ctx0 || device_ctx0->type != AV_HWDEVICE_TYPE_RKMPP) { + if (avctx->hw_device_ctx) { + device_ref0 = avctx->hw_device_ctx; + device_ctx0 = (AVHWDeviceContext *)device_ref0->data; + } + if (!device_ctx0 || device_ctx0->type != AV_HWDEVICE_TYPE_RKMPP) { + av_log(avctx, AV_LOG_ERROR, "No RKMPP hardware context provided\n"); + return AVERROR(EINVAL); + } + } + + hwfc_pat_ref = av_hwframe_ctx_alloc(device_ref0); + if (!hwfc_pat_ref) + return AVERROR(ENOMEM); + + hwfc_pat = (AVHWFramesContext *)hwfc_pat_ref->data; + hwfc_pat->format = AV_PIX_FMT_DRM_PRIME; + hwfc_pat->sw_format = hwfc_in1->sw_format; + hwfc_pat->width = inlink0->w; + hwfc_pat->height = inlink0->h; + + ret = av_hwframe_ctx_init(hwfc_pat_ref); + if (ret < 0) { + av_log(avctx, AV_LOG_ERROR, "Error creating frames_ctx for pat preproc: %d\n", ret); + av_buffer_unref(&hwfc_pat_ref); + return ret; + } + + av_buffer_unref(&r->pat_preproc_hwframes_ctx); + r->pat_preproc_hwframes_ctx = hwfc_pat_ref; + + return 0; +} + +static av_cold int verify_rga_frame_info(AVFilterContext *avctx, + RGAFrameInfo *src, RGAFrameInfo *dst, RGAFrameInfo *pat) +{ + RKRGAContext *r = avctx->priv; + float scale_ratio_min, scale_ratio_max; + float scale_ratio_w, scale_ratio_h; + int ret; + + if (!src || !dst) + return AVERROR(EINVAL); + + scale_ratio_w = (float)dst->act_w / (float)src->act_w; + scale_ratio_h = (float)dst->act_h / (float)src->act_h; + + /* P010 requires RGA3 */ + if (!r->has_rga3 && + (src->pix_fmt == AV_PIX_FMT_P010 || + dst->pix_fmt == AV_PIX_FMT_P010)) { + av_log(avctx, AV_LOG_ERROR, "'%s' is only supported by RGA3\n", + av_get_pix_fmt_name(AV_PIX_FMT_P010)); + return AVERROR(ENOSYS); + } + /* P210 requires RGA3 */ + if (!r->has_rga3 && + (src->pix_fmt == AV_PIX_FMT_P210 || + dst->pix_fmt == AV_PIX_FMT_P210)) { + av_log(avctx, AV_LOG_ERROR, "'%s' is only supported by RGA3\n", + av_get_pix_fmt_name(AV_PIX_FMT_P210)); + return AVERROR(ENOSYS); + } + /* NV24/NV42 requires RGA2-Pro */ + if (!r->has_rga2p && + (src->pix_fmt == AV_PIX_FMT_NV24 || + src->pix_fmt == AV_PIX_FMT_NV42 || + dst->pix_fmt == AV_PIX_FMT_NV24 || + dst->pix_fmt == AV_PIX_FMT_NV42)) { + av_log(avctx, AV_LOG_ERROR, "'%s' and '%s' are only supported by RGA2-Pro\n", + av_get_pix_fmt_name(AV_PIX_FMT_NV24), + av_get_pix_fmt_name(AV_PIX_FMT_NV42)); + return AVERROR(ENOSYS); + } + /* Input formats that requires RGA2 */ + if (!r->has_rga2 && + (src->pix_fmt == AV_PIX_FMT_GRAY8 || + src->pix_fmt == AV_PIX_FMT_YUV420P || + src->pix_fmt == AV_PIX_FMT_YUVJ420P || + src->pix_fmt == AV_PIX_FMT_YUV422P || + src->pix_fmt == AV_PIX_FMT_YUVJ422P || + src->pix_fmt == AV_PIX_FMT_RGB555LE || + src->pix_fmt == AV_PIX_FMT_BGR555LE)) { + av_log(avctx, AV_LOG_ERROR, "'%s' as input is only supported by RGA2\n", + av_get_pix_fmt_name(src->pix_fmt)); + return AVERROR(ENOSYS); + } + /* Output formats that requires RGA2 */ + if (!r->has_rga2 && + (dst->pix_fmt == AV_PIX_FMT_GRAY8 || + dst->pix_fmt == AV_PIX_FMT_YUV420P || + dst->pix_fmt == AV_PIX_FMT_YUVJ420P || + dst->pix_fmt == AV_PIX_FMT_YUV422P || + dst->pix_fmt == AV_PIX_FMT_YUVJ422P || + dst->pix_fmt == AV_PIX_FMT_RGB555LE || + dst->pix_fmt == AV_PIX_FMT_BGR555LE || + dst->pix_fmt == AV_PIX_FMT_ARGB || + dst->pix_fmt == AV_PIX_FMT_0RGB || + dst->pix_fmt == AV_PIX_FMT_ABGR || + dst->pix_fmt == AV_PIX_FMT_0BGR)) { + av_log(avctx, AV_LOG_ERROR, "'%s' as output is only supported by RGA2\n", + av_get_pix_fmt_name(dst->pix_fmt)); + return AVERROR(ENOSYS); + } + /* non-YUVJ format to YUVJ format is not supported */ + if ((dst->pix_fmt == AV_PIX_FMT_YUVJ420P || + dst->pix_fmt == AV_PIX_FMT_YUVJ422P) && + (src->pix_fmt != AV_PIX_FMT_YUVJ420P && + src->pix_fmt != AV_PIX_FMT_YUVJ422P)) { + av_log(avctx, AV_LOG_ERROR, "'%s' to '%s' is not supported\n", + av_get_pix_fmt_name(src->pix_fmt), + av_get_pix_fmt_name(dst->pix_fmt)); + return AVERROR(ENOSYS); + } + /* P010/P210 requires RGA3 but it can't handle certain formats */ + if ((src->pix_fmt == AV_PIX_FMT_P010 || + src->pix_fmt == AV_PIX_FMT_P210) && + (dst->pix_fmt == AV_PIX_FMT_GRAY8 || + dst->pix_fmt == AV_PIX_FMT_YUV420P || + dst->pix_fmt == AV_PIX_FMT_YUVJ420P || + dst->pix_fmt == AV_PIX_FMT_YUV422P || + dst->pix_fmt == AV_PIX_FMT_YUVJ422P || + dst->pix_fmt == AV_PIX_FMT_RGB555LE || + dst->pix_fmt == AV_PIX_FMT_BGR555LE || + dst->pix_fmt == AV_PIX_FMT_ARGB || + dst->pix_fmt == AV_PIX_FMT_0RGB || + dst->pix_fmt == AV_PIX_FMT_ABGR || + dst->pix_fmt == AV_PIX_FMT_0BGR)) { + av_log(avctx, AV_LOG_ERROR, "'%s' to '%s' is not supported\n", + av_get_pix_fmt_name(src->pix_fmt), + av_get_pix_fmt_name(dst->pix_fmt)); + return AVERROR(ENOSYS); + } + /* RGA3 only format to RGA2 only format is not supported */ + if ((dst->pix_fmt == AV_PIX_FMT_P010 || + dst->pix_fmt == AV_PIX_FMT_P210) && + (src->pix_fmt == AV_PIX_FMT_GRAY8 || + src->pix_fmt == AV_PIX_FMT_YUV420P || + src->pix_fmt == AV_PIX_FMT_YUVJ420P || + src->pix_fmt == AV_PIX_FMT_YUV422P || + src->pix_fmt == AV_PIX_FMT_YUVJ422P || + src->pix_fmt == AV_PIX_FMT_RGB555LE || + src->pix_fmt == AV_PIX_FMT_BGR555LE)) { + av_log(avctx, AV_LOG_ERROR, "'%s' to '%s' is not supported\n", + av_get_pix_fmt_name(src->pix_fmt), + av_get_pix_fmt_name(dst->pix_fmt)); + return AVERROR(ENOSYS); + } + + if (src->pix_fmt == AV_PIX_FMT_GRAY8 || + src->pix_fmt == AV_PIX_FMT_YUV420P || + src->pix_fmt == AV_PIX_FMT_YUVJ420P || + src->pix_fmt == AV_PIX_FMT_YUV422P || + src->pix_fmt == AV_PIX_FMT_YUVJ422P || + src->pix_fmt == AV_PIX_FMT_NV24 || + src->pix_fmt == AV_PIX_FMT_NV42 || + src->pix_fmt == AV_PIX_FMT_RGB555LE || + src->pix_fmt == AV_PIX_FMT_BGR555LE || + dst->pix_fmt == AV_PIX_FMT_GRAY8 || + dst->pix_fmt == AV_PIX_FMT_YUV420P || + dst->pix_fmt == AV_PIX_FMT_YUVJ420P || + dst->pix_fmt == AV_PIX_FMT_YUV422P || + dst->pix_fmt == AV_PIX_FMT_YUVJ422P || + dst->pix_fmt == AV_PIX_FMT_NV24 || + dst->pix_fmt == AV_PIX_FMT_NV42 || + dst->pix_fmt == AV_PIX_FMT_RGB555LE || + dst->pix_fmt == AV_PIX_FMT_BGR555LE || + dst->pix_fmt == AV_PIX_FMT_ARGB || + dst->pix_fmt == AV_PIX_FMT_0RGB || + dst->pix_fmt == AV_PIX_FMT_ABGR || + dst->pix_fmt == AV_PIX_FMT_0BGR) { + r->is_rga2_used = 1; + } + + r->is_rga2_used = r->is_rga2_used || !r->has_rga3; + if (r->has_rga3) { + if (scale_ratio_w < 0.125f || + scale_ratio_w > 8.0f || + scale_ratio_h < 0.125f || + scale_ratio_h > 8.0f) { + r->is_rga2_used = 1; + } + if (src->act_w < 68 || + src->act_w > 8176 || + src->act_h > 8176 || + dst->act_w < 68) { + r->is_rga2_used = 1; + } + if (pat && (pat->act_w < 68 || + pat->act_w > 8176 || + pat->act_h > 8176)) { + r->is_rga2_used = 1; + } + } + + if ((ret = verify_rga_frame_info_io_dynamic(avctx, src, dst)) < 0) + return ret; + + if (r->is_rga2_used) { + r->scheduler_core = 0x4; + if (r->has_rga2p) + r->scheduler_core |= 0x8; + } + + /* Prioritize RGA3 on multicore RGA hw to avoid dma32 & algorithm quirks as much as possible */ + if (r->has_rga3 && r->has_rga2e && !r->is_rga2_used && + (r->scheduler_core == 0 || avctx->nb_inputs > 1 || + scale_ratio_w != 1.0f || scale_ratio_h != 1.0f || + src->crop || src->uncompact_10b_msb || dst->uncompact_10b_msb)) { + r->scheduler_core = 0x3; + } + + scale_ratio_max = 16.0f; + if ((r->is_rga2_used && r->has_rga2l) || + (!r->is_rga2_used && r->has_rga3 && !r->has_rga2) || + (r->scheduler_core > 0 && r->scheduler_core == (r->scheduler_core & 0x3))) { + scale_ratio_max = 8.0f; + } + scale_ratio_min = 1.0f / scale_ratio_max; + + if (scale_ratio_w < scale_ratio_min || scale_ratio_w > scale_ratio_max || + scale_ratio_h < scale_ratio_min || scale_ratio_h > scale_ratio_max) { + av_log(avctx, AV_LOG_ERROR, "RGA scale ratio (%.04fx%.04f) exceeds %.04f ~ %.04f.\n", + scale_ratio_w, scale_ratio_h, scale_ratio_min, scale_ratio_max); + return AVERROR(EINVAL); + } + + return 0; +} + +static av_cold int fill_rga_frame_info_by_link(AVFilterContext *avctx, + RGAFrameInfo *info, + AVFilterLink *link, + int nb_link, int is_inlink) +{ + AVHWFramesContext *hwfc; + RKRGAContext *r = avctx->priv; + + if (!link->hw_frames_ctx || link->format != AV_PIX_FMT_DRM_PRIME) + return AVERROR(EINVAL); + + hwfc = (AVHWFramesContext *)link->hw_frames_ctx->data; + + if (!map_av_to_rga_format(hwfc->sw_format, &info->rga_fmt, (is_inlink && nb_link > 0))) { + av_log(avctx, AV_LOG_ERROR, "Unsupported '%s' pad %d format: '%s'\n", + (is_inlink ? "input" : "output"), nb_link, + av_get_pix_fmt_name(hwfc->sw_format)); + return AVERROR(ENOSYS); + } + + info->pix_fmt = hwfc->sw_format; + info->pix_desc = av_pix_fmt_desc_get(info->pix_fmt); + info->bytes_pp = av_get_padded_bits_per_pixel(info->pix_desc) / 8.0f; + + info->act_x = 0; + info->act_y = 0; + info->act_w = link->w; + info->act_h = link->h; + + /* The w/h of RGA YUV image needs to be 2 aligned */ + if (!(info->pix_desc->flags & AV_PIX_FMT_FLAG_RGB)) { + info->act_w = ALIGN_DOWN(info->act_w, RK_RGA_YUV_ALIGN); + info->act_h = ALIGN_DOWN(info->act_h, RK_RGA_YUV_ALIGN); + } + + info->uncompact_10b_msb = info->pix_fmt == AV_PIX_FMT_P010 || + info->pix_fmt == AV_PIX_FMT_P210; + + if (link->w * link->h > (3840 * 2160 * 3)) + r->async_depth = FFMIN(r->async_depth, 1); + + return 0; +} + +av_cold int ff_rkrga_init(AVFilterContext *avctx, RKRGAParam *param) +{ + RKRGAContext *r = avctx->priv; + int i, ret; + int rga_core_mask = 0x7; + const char *rga_ver = querystring(RGA_VERSION); + + r->got_frame = 0; + + r->has_rga2 = !!strstr(rga_ver, "RGA_2"); + r->has_rga2l = !!strstr(rga_ver, "RGA_2_lite"); + r->has_rga2e = !!strstr(rga_ver, "RGA_2_Enhance"); + r->has_rga2p = !!strstr(rga_ver, "RGA_2_PRO"); + r->has_rga3 = !!strstr(rga_ver, "RGA_3"); + + if (!(r->has_rga2 || r->has_rga3)) { + av_log(avctx, AV_LOG_ERROR, "No RGA2/RGA3 hw available\n"); + return AVERROR(ENOSYS); + } + + if (r->has_rga2p) + rga_core_mask = 0xf; + + /* RGA core */ + if (r->scheduler_core && !(r->has_rga2 && r->has_rga3) && !r->has_rga2p) { + av_log(avctx, AV_LOG_WARNING, "Scheduler core cannot be set on non-multicore RGA hw, ignoring\n"); + r->scheduler_core = 0; + } + if (r->scheduler_core && r->scheduler_core != (r->scheduler_core & rga_core_mask)) { + av_log(avctx, AV_LOG_WARNING, "Invalid scheduler core set, ignoring\n"); + r->scheduler_core = 0; + } + if (r->scheduler_core && r->scheduler_core == (r->scheduler_core & 0x3)) + r->has_rga2 = r->has_rga2l = r->has_rga2e = r->has_rga2p = 0; + if (r->scheduler_core == 0x4 && !r->has_rga2p) + r->has_rga3 = 0; + + r->filter_frame = param->filter_frame; + if (!r->filter_frame) + r->filter_frame = ff_filter_frame; + r->out_sw_format = param->out_sw_format; + + /* OUT hwfc */ + ret = init_hwframes_ctx(avctx); + if (ret < 0) + goto fail; + + /* IN RGAFrameInfo */ + r->in_rga_frame_infos = av_calloc(avctx->nb_inputs, sizeof(*r->in_rga_frame_infos)); + if (!r->in_rga_frame_infos) { + ret = AVERROR(ENOMEM); + goto fail; + } + for (i = 0; i < avctx->nb_inputs; i++) { + ret = fill_rga_frame_info_by_link(avctx, &r->in_rga_frame_infos[i], avctx->inputs[i], i, 1); + if (ret < 0) + goto fail; + } + if (avctx->nb_inputs == 1) { + r->in_rga_frame_infos[0].rotate_mode = param->in_rotate_mode; + + if (param->in_crop) { + /* The x/y/w/h of RGA YUV image needs to be 2 aligned */ + if (!(r->in_rga_frame_infos[0].pix_desc->flags & AV_PIX_FMT_FLAG_RGB)) { + param->in_crop_x = ALIGN_DOWN(param->in_crop_x, RK_RGA_YUV_ALIGN); + param->in_crop_y = ALIGN_DOWN(param->in_crop_y, RK_RGA_YUV_ALIGN); + param->in_crop_w = ALIGN_DOWN(param->in_crop_w, RK_RGA_YUV_ALIGN); + param->in_crop_h = ALIGN_DOWN(param->in_crop_h, RK_RGA_YUV_ALIGN); + } + r->in_rga_frame_infos[0].crop = 1; + r->in_rga_frame_infos[0].act_x = param->in_crop_x; + r->in_rga_frame_infos[0].act_y = param->in_crop_y; + r->in_rga_frame_infos[0].act_w = param->in_crop_w; + r->in_rga_frame_infos[0].act_h = param->in_crop_h; + } + } + if (avctx->nb_inputs > 1) { + int need_premultiply = 0; + + if (r->in_rga_frame_infos[1].pix_desc->flags & AV_PIX_FMT_FLAG_ALPHA) + need_premultiply = param->in_alpha_format == 0; + + /* IM_ALPHA_BLEND_DST_OVER */ + if (param->in_global_alpha > 0 && param->in_global_alpha < 0xff) { + r->in_rga_frame_infos[0].blend_mode = need_premultiply ? (0x4 | (1 << 12)) : 0x4; + r->in_rga_frame_infos[0].blend_mode |= (param->in_global_alpha & 0xff) << 16; /* fg_global_alpha */ + r->in_rga_frame_infos[0].blend_mode |= 0xff << 24; /* bg_global_alpha */ + } else + r->in_rga_frame_infos[0].blend_mode = need_premultiply ? 0x504 : 0x501; + + r->in_rga_frame_infos[1].overlay_x = FFMAX(param->overlay_x, 0); + r->in_rga_frame_infos[1].overlay_y = FFMAX(param->overlay_y, 0); + + r->is_overlay_offset_valid = (param->overlay_x < r->in_rga_frame_infos[0].act_w - 2) && + (param->overlay_y < r->in_rga_frame_infos[0].act_h - 2); + if (r->is_overlay_offset_valid) + init_pat_preproc_hwframes_ctx(avctx); + } + + /* OUT RGAFrameInfo */ + ret = fill_rga_frame_info_by_link(avctx, &r->out_rga_frame_info, avctx->outputs[0], 0, 0); + if (ret < 0) + goto fail; + + /* Pre-check RGAFrameInfo */ + ret = verify_rga_frame_info(avctx, &r->in_rga_frame_infos[0], + &r->out_rga_frame_info, + (avctx->nb_inputs > 1 ? &r->in_rga_frame_infos[1] : NULL)); + if (ret < 0) + goto fail; + + r->out_rga_frame_info.scheduler_core = r->scheduler_core; + + /* keep fifo size at least 1. Even when async_depth is 0, fifo is used. */ + r->async_fifo = av_fifo_alloc2(r->async_depth + 1, sizeof(RGAAsyncFrame), 0); + if (!r->async_fifo) { + ret = AVERROR(ENOMEM); + goto fail; + } + + return 0; + +fail: + ff_rkrga_close(avctx); + return ret; +} + +static void set_rga_async_frame_lock_status(RGAAsyncFrame *frame, int lock) +{ + int status = !!lock; + + if (!frame) + return; + + if (frame->src) + frame->src->locked = status; + if (frame->dst) + frame->dst->locked = status; + if (frame->pat) + frame->pat->locked = status; +} + +static void rga_drain_fifo(RKRGAContext *r) +{ + RGAAsyncFrame aframe; + + while (r->async_fifo && av_fifo_read(r->async_fifo, &aframe, 1) >= 0) { + if (imsync(aframe.dst->info.out_fence_fd) != IM_STATUS_SUCCESS) + av_log(NULL, AV_LOG_WARNING, "RGA sync failed\n"); + + set_rga_async_frame_lock_status(&aframe, 0); + } +} + +av_cold int ff_rkrga_close(AVFilterContext *avctx) +{ + RKRGAContext *r = avctx->priv; + + /* Drain the fifo during filter reset */ + rga_drain_fifo(r); + + clear_frame_list(&r->src_frame_list); + clear_frame_list(&r->dst_frame_list); + clear_frame_list(&r->pat_frame_list); + + clear_frame_list(&r->pat_preproc_frame_list); + + if (r->in_rga_frame_infos) + av_freep(&r->in_rga_frame_infos); + + av_fifo_freep2(&r->async_fifo); + + av_buffer_unref(&r->pat_preproc_hwframes_ctx); + + return 0; +} + +static int call_rkrga_blit(AVFilterContext *avctx, + rga_info_t *src_info, + rga_info_t *dst_info, + rga_info_t *pat_info) +{ + int ret; + + if (!src_info || !dst_info) + return AVERROR(EINVAL); + +#define PRINT_RGA_INFO(ctx, info, name) do { \ + if (info && name) \ + av_log(ctx, AV_LOG_DEBUG, "RGA %s | fd:%d mmu:%d rd:%d csc:%d | x:%d y:%d w:%d h:%d ws:%d hs:%d fmt:0x%x\n", \ + name, info->fd, info->mmuFlag, (info->rd_mode >> 1), info->color_space_mode, info->rect.xoffset, info->rect.yoffset, \ + info->rect.width, info->rect.height, info->rect.wstride, info->rect.hstride, (info->rect.format >> 8)); \ +} while (0) + + PRINT_RGA_INFO(avctx, src_info, "src"); + PRINT_RGA_INFO(avctx, dst_info, "dst"); + PRINT_RGA_INFO(avctx, pat_info, "pat"); +#undef PRINT_RGA_INFO + + if ((ret = c_RkRgaBlit(src_info, dst_info, pat_info)) != 0) { + av_log(avctx, AV_LOG_ERROR, "RGA blit failed: %d\n", ret); + return AVERROR_EXTERNAL; + } + if (dst_info->sync_mode == RGA_BLIT_ASYNC && + dst_info->out_fence_fd <= 0) { + av_log(avctx, AV_LOG_ERROR, "RGA async blit returned invalid fence_fd: %d\n", + dst_info->out_fence_fd); + return AVERROR_EXTERNAL; + } + + return 0; +} + +int ff_rkrga_filter_frame(RKRGAContext *r, + AVFilterLink *inlink_src, AVFrame *picref_src, + AVFilterLink *inlink_pat, AVFrame *picref_pat) +{ + AVFilterContext *ctx = inlink_src->dst; + AVFilterLink *outlink = ctx->outputs[0]; + RGAAsyncFrame aframe; + RGAFrame *src_frame = NULL; + RGAFrame *dst_frame = NULL; + RGAFrame *pat_frame = NULL; + int ret, filter_ret; + int do_overlay = ctx->nb_inputs > 1 && + r->is_overlay_offset_valid && + inlink_pat && picref_pat; + + /* Sync & Drain */ + while (r->eof && av_fifo_read(r->async_fifo, &aframe, 1) >= 0) { + if (imsync(aframe.dst->info.out_fence_fd) != IM_STATUS_SUCCESS) + av_log(ctx, AV_LOG_WARNING, "RGA sync failed\n"); + + set_rga_async_frame_lock_status(&aframe, 0); + + filter_ret = r->filter_frame(outlink, aframe.dst->frame); + if (filter_ret < 0) { + av_frame_free(&aframe.dst->frame); + return filter_ret; + } + aframe.dst->queued--; + r->got_frame = 1; + aframe.dst->frame = NULL; + } + + if (!picref_src) + return 0; + + /* SRC */ + if (!(src_frame = submit_frame(r, inlink_src, picref_src, do_overlay, 0))) { + av_log(ctx, AV_LOG_ERROR, "Failed to submit frame on input: %d\n", + FF_INLINK_IDX(inlink_src)); + return AVERROR(ENOMEM); + } + + /* DST */ + if (!(dst_frame = query_frame(r, outlink, src_frame->frame, picref_pat, 0))) { + av_log(ctx, AV_LOG_ERROR, "Failed to query an output frame\n"); + return AVERROR(ENOMEM); + } + + /* PAT */ + if (do_overlay) { + RGAFrameInfo *in0_info = &r->in_rga_frame_infos[0]; + RGAFrameInfo *in1_info = &r->in_rga_frame_infos[1]; + RGAFrameInfo *out_info = &r->out_rga_frame_info; + RGAFrame *pat_in = NULL; + RGAFrame *pat_out = NULL; + + /* translate PAT from top-left to (x,y) on a new image with the same size of SRC */ + if (in1_info->act_w != in0_info->act_w || + in1_info->act_h != in0_info->act_h || + in1_info->overlay_x > 0 || + in1_info->overlay_y > 0) { + if (!(pat_in = submit_frame(r, inlink_pat, picref_pat, 0, 1))) { + av_log(ctx, AV_LOG_ERROR, "Failed to submit frame on input: %d\n", + FF_INLINK_IDX(inlink_pat)); + return AVERROR(ENOMEM); + } + if (!(pat_out = query_frame(r, outlink, picref_pat, NULL, 1))) { + av_log(ctx, AV_LOG_ERROR, "Failed to query an output frame\n"); + return AVERROR(ENOMEM); + } + dst_frame->info.core = out_info->scheduler_core; + + pat_out->info.priority = 1; + pat_out->info.core = dst_frame->info.core; + pat_out->info.sync_mode = RGA_BLIT_SYNC; + + /* Sync Blit Pre-Proc */ + ret = call_rkrga_blit(ctx, &pat_in->info, &pat_out->info, NULL); + if (ret < 0) + return ret; + + pat_out->info.rect.xoffset = 0; + pat_out->info.rect.yoffset = 0; + pat_out->info.rect.width = in0_info->act_w; + pat_out->info.rect.height = in0_info->act_h; + + pat_frame = pat_out; + } + + if (!pat_frame && !(pat_frame = submit_frame(r, inlink_pat, picref_pat, 0, 0))) { + av_log(ctx, AV_LOG_ERROR, "Failed to submit frame on input: %d\n", + FF_INLINK_IDX(inlink_pat)); + return AVERROR(ENOMEM); + } + dst_frame->info.core = out_info->scheduler_core; + } + + /* Async Blit */ + ret = call_rkrga_blit(ctx, + &src_frame->info, + &dst_frame->info, + pat_frame ? &pat_frame->info : NULL); + if (ret < 0) + return ret; + + dst_frame->queued++; + aframe = (RGAAsyncFrame){ src_frame, dst_frame, pat_frame }; + set_rga_async_frame_lock_status(&aframe, 1); + av_fifo_write(r->async_fifo, &aframe, 1); + + /* Sync & Retrieve */ + if (av_fifo_can_read(r->async_fifo) > r->async_depth) { + av_fifo_read(r->async_fifo, &aframe, 1); + if (imsync(aframe.dst->info.out_fence_fd) != IM_STATUS_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "RGA sync failed\n"); + return AVERROR_EXTERNAL; + } + set_rga_async_frame_lock_status(&aframe, 0); + + filter_ret = r->filter_frame(outlink, aframe.dst->frame); + if (filter_ret < 0) { + av_frame_free(&aframe.dst->frame); + return filter_ret; + } + aframe.dst->queued--; + r->got_frame = 1; + aframe.dst->frame = NULL; + } + + return 0; +} diff --git a/libavfilter/rkrga_common.h b/libavfilter/rkrga_common.h new file mode 100644 index 0000000000..9e88477330 --- /dev/null +++ b/libavfilter/rkrga_common.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2023 NyanMisaka + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Rockchip RGA (2D Raster Graphic Acceleration) base function + */ + +#ifndef AVFILTER_RKRGA_COMMON_H +#define AVFILTER_RKRGA_COMMON_H + +#include +#include + +#include "avfilter.h" +#include "libavutil/fifo.h" +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_rkmpp.h" + +#define RK_RGA_YUV_ALIGN 2 +#define RK_RGA_AFBC_16x16_STRIDE_ALIGN 16 +#define RK_RGA_RFBC_64x4_STRIDE_ALIGN_W 64 +#define RK_RGA_RFBC_64x4_STRIDE_ALIGN_H 4 + +#define ALIGN_DOWN(a, b) ((a) & ~((b)-1)) +#define FF_INLINK_IDX(link) ((int)((link)->dstpad - (link)->dst->input_pads)) +#define FF_OUTLINK_IDX(link) ((int)((link)->srcpad - (link)->src->output_pads)) + +typedef struct RGAFrame { + AVFrame *frame; + rga_info_t info; + struct RGAFrame *next; + int queued; + int locked; +} RGAFrame; + +typedef struct RGAFrameInfo { + enum _Rga_SURF_FORMAT rga_fmt; + enum AVPixelFormat pix_fmt; + const AVPixFmtDescriptor *pix_desc; + float bytes_pp; + int act_x; + int act_y; + int act_w; + int act_h; + int uncompact_10b_msb; + int rotate_mode; + int blend_mode; + int crop; + int scheduler_core; + int overlay_x; + int overlay_y; +} RGAFrameInfo; + +typedef struct RKRGAContext { + const AVClass *class; + + int (*filter_frame) (AVFilterLink *outlink, AVFrame *frame); + enum AVPixelFormat out_sw_format; + + RGAFrame *src_frame_list; + RGAFrame *dst_frame_list; + RGAFrame *pat_frame_list; + + AVBufferRef *pat_preproc_hwframes_ctx; + RGAFrame *pat_preproc_frame_list; + + RGAFrameInfo *in_rga_frame_infos; + RGAFrameInfo out_rga_frame_info; + + int scheduler_core; + int async_depth; + int afbc_out; + + int has_rga2; + int has_rga2l; + int has_rga2e; + int has_rga2p; + int has_rga3; + int is_rga2_used; + int is_overlay_offset_valid; + + int eof; + int got_frame; + + AVFifo *async_fifo; +} RKRGAContext; + +typedef struct RKRGAParam { + int (*filter_frame)(AVFilterLink *outlink, AVFrame *frame); + + enum AVPixelFormat out_sw_format; + + int in_rotate_mode; + int in_global_alpha; + int in_alpha_format; + + int in_crop; + int in_crop_x; + int in_crop_y; + int in_crop_w; + int in_crop_h; + + int overlay_x; + int overlay_y; +} RKRGAParam; + +int ff_rkrga_init(AVFilterContext *avctx, RKRGAParam *param); +int ff_rkrga_close(AVFilterContext *avctx); +int ff_rkrga_filter_frame(RKRGAContext *r, + AVFilterLink *inlink_src, AVFrame *picref_src, + AVFilterLink *inlink_pat, AVFrame *picref_pat); + +#endif /* AVFILTER_RKRGA_COMMON_H */ diff --git a/libavfilter/vf_overlay_rkrga.c b/libavfilter/vf_overlay_rkrga.c new file mode 100644 index 0000000000..955b8d25c9 --- /dev/null +++ b/libavfilter/vf_overlay_rkrga.c @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2023 NyanMisaka + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Rockchip RGA (2D Raster Graphic Acceleration) video compositor + */ + +#include "libavutil/common.h" +#include "libavutil/eval.h" +#include "libavutil/internal.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" + +#include "filters.h" +#include "framesync.h" + +#include "rkrga_common.h" + +enum var_name { + VAR_MAIN_W, VAR_MW, + VAR_MAIN_H, VAR_MH, + VAR_OVERLAY_W, VAR_OW, + VAR_OVERLAY_H, VAR_OH, + VAR_OVERLAY_X, VAR_OX, + VAR_OVERLAY_Y, VAR_OY, + VAR_VARS_NB +}; + +typedef struct RGAOverlayContext { + RKRGAContext rga; + + FFFrameSync fs; + + double var_values[VAR_VARS_NB]; + char *overlay_ox, *overlay_oy; + int global_alpha; + int alpha_format; + enum AVPixelFormat format; +} RGAOverlayContext; + +static const char *const var_names[] = { + "main_w", "W", /* input width of the main layer */ + "main_h", "H", /* input height of the main layer */ + "overlay_w", "w", /* input width of the overlay layer */ + "overlay_h", "h", /* input height of the overlay layer */ + "overlay_x", "x", /* x position of the overlay layer inside of main */ + "overlay_y", "y", /* y position of the overlay layer inside of main */ + NULL +}; + +static int eval_expr(AVFilterContext *ctx) +{ + RGAOverlayContext *r = ctx->priv; + double *var_values = r->var_values; + int ret = 0; + AVExpr *ox_expr = NULL, *oy_expr = NULL; + AVExpr *ow_expr = NULL, *oh_expr = NULL; + +#define PASS_EXPR(e, s) {\ + ret = av_expr_parse(&e, s, var_names, NULL, NULL, NULL, NULL, 0, ctx); \ + if (ret < 0) {\ + av_log(ctx, AV_LOG_ERROR, "Error when passing '%s'.\n", s);\ + goto release;\ + }\ +} + PASS_EXPR(ox_expr, r->overlay_ox); + PASS_EXPR(oy_expr, r->overlay_oy); + PASS_EXPR(ow_expr, "overlay_w"); + PASS_EXPR(oh_expr, "overlay_h"); +#undef PASS_EXPR + + var_values[VAR_OVERLAY_W] = + var_values[VAR_OW] = av_expr_eval(ow_expr, var_values, NULL); + var_values[VAR_OVERLAY_H] = + var_values[VAR_OH] = av_expr_eval(oh_expr, var_values, NULL); + + /* calc again in case ow is relative to oh */ + var_values[VAR_OVERLAY_W] = + var_values[VAR_OW] = av_expr_eval(ow_expr, var_values, NULL); + + var_values[VAR_OVERLAY_X] = + var_values[VAR_OX] = av_expr_eval(ox_expr, var_values, NULL); + var_values[VAR_OVERLAY_Y] = + var_values[VAR_OY] = av_expr_eval(oy_expr, var_values, NULL); + + /* calc again in case ox is relative to oy */ + var_values[VAR_OVERLAY_X] = + var_values[VAR_OX] = av_expr_eval(ox_expr, var_values, NULL); + +release: + av_expr_free(ox_expr); + av_expr_free(oy_expr); + av_expr_free(ow_expr); + av_expr_free(oh_expr); + + return ret; +} + +static av_cold int set_size_info(AVFilterContext *ctx, + AVFilterLink *inlink_main, + AVFilterLink *inlink_overlay, + AVFilterLink *outlink) +{ + RGAOverlayContext *r = ctx->priv; + int ret; + + if (inlink_main->w < 2 || inlink_main->w > 8192 || + inlink_main->h < 2 || inlink_main->h > 8192 || + inlink_overlay->w < 2 || inlink_overlay->w > 8192 || + inlink_overlay->h < 2 || inlink_overlay->h > 8192) { + av_log(ctx, AV_LOG_ERROR, "Supported input size is range from 2x2 ~ 8192x8192\n"); + return AVERROR(EINVAL); + } + + r->var_values[VAR_MAIN_W] = + r->var_values[VAR_MW] = inlink_main->w; + r->var_values[VAR_MAIN_H] = + r->var_values[VAR_MH] = inlink_main->h; + + r->var_values[VAR_OVERLAY_W] = inlink_overlay->w; + r->var_values[VAR_OVERLAY_H] = inlink_overlay->h; + + if ((ret = eval_expr(ctx)) < 0) + return ret; + + outlink->w = r->var_values[VAR_MW]; + outlink->h = r->var_values[VAR_MH]; + if (outlink->w < 2 || outlink->w > 8192 || + outlink->h < 2 || outlink->h > 8192) { + av_log(ctx, AV_LOG_ERROR, "Supported output size is range from 2x2 ~ 8192x8192\n"); + return AVERROR(EINVAL); + } + + if (inlink_main->sample_aspect_ratio.num) + outlink->sample_aspect_ratio = av_mul_q((AVRational){outlink->h * inlink_main->w, + outlink->w * inlink_main->h}, + inlink_main->sample_aspect_ratio); + else + outlink->sample_aspect_ratio = inlink_main->sample_aspect_ratio; + + return 0; +} + +static av_cold int rgaoverlay_config_props(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + RGAOverlayContext *r = ctx->priv; + AVFilterLink *inlink_main = ctx->inputs[0]; + AVFilterLink *inlink_overlay = ctx->inputs[1]; + AVHWFramesContext *frames_ctx_main; + AVHWFramesContext *frames_ctx_overlay; + enum AVPixelFormat in_format_main; + enum AVPixelFormat in_format_overlay; + enum AVPixelFormat out_format; + int ret; + + RKRGAParam param = { NULL }; + + if (!inlink_main->hw_frames_ctx) { + av_log(ctx, AV_LOG_ERROR, "No hw context provided on main input\n"); + return AVERROR(EINVAL); + } + frames_ctx_main = (AVHWFramesContext *)inlink_main->hw_frames_ctx->data; + in_format_main = frames_ctx_main->sw_format; + out_format = (r->format == AV_PIX_FMT_NONE) ? in_format_main : r->format; + + if (!inlink_overlay->hw_frames_ctx) { + av_log(ctx, AV_LOG_ERROR, "No hw context provided on overlay input\n"); + return AVERROR(EINVAL); + } + frames_ctx_overlay = (AVHWFramesContext *)inlink_overlay->hw_frames_ctx->data; + in_format_overlay = frames_ctx_overlay->sw_format; + + ret = set_size_info(ctx, inlink_main, inlink_overlay, outlink); + if (ret < 0) + return ret; + + param.filter_frame = NULL; + param.out_sw_format = out_format; + param.in_global_alpha = r->global_alpha; + param.in_alpha_format = r->alpha_format; + param.overlay_x = r->var_values[VAR_OX]; + param.overlay_y = r->var_values[VAR_OY]; + + ret = ff_rkrga_init(ctx, ¶m); + if (ret < 0) + return ret; + + av_log(ctx, AV_LOG_VERBOSE, "w:%d h:%d fmt:%s + w:%d h:%d fmt:%s (x:%d y:%d) -> w:%d h:%d fmt:%s\n", + inlink_main->w, inlink_main->h, av_get_pix_fmt_name(in_format_main), + inlink_overlay->w, inlink_overlay->h, av_get_pix_fmt_name(in_format_overlay), + param.overlay_x, param.overlay_y, outlink->w, outlink->h, av_get_pix_fmt_name(out_format)); + + ret = ff_framesync_init_dualinput(&r->fs, ctx); + if (ret < 0) + return ret; + + r->fs.time_base = outlink->time_base = inlink_main->time_base; + + ret = ff_framesync_configure(&r->fs); + if (ret < 0) + return ret; + + return 0; +} + +static int rgaoverlay_on_event(FFFrameSync *fs) +{ + AVFilterContext *ctx = fs->parent; + AVFilterLink *inlink_main = ctx->inputs[0]; + AVFilterLink *inlink_overlay = ctx->inputs[1]; + AVFrame *in_main = NULL, *in_overlay = NULL; + int ret; + + RGAOverlayContext *r = ctx->priv; + + ret = ff_framesync_get_frame(fs, 0, &in_main, 0); + if (ret < 0) + return ret; + ret = ff_framesync_get_frame(fs, 1, &in_overlay, 0); + if (ret < 0) + return ret; + + if (!in_main) + return AVERROR_BUG; + + return ff_rkrga_filter_frame(&r->rga, + inlink_main, in_main, + inlink_overlay, in_overlay); +} + +static av_cold int rgaoverlay_init(AVFilterContext *ctx) +{ + RGAOverlayContext *r = ctx->priv; + + r->fs.on_event = &rgaoverlay_on_event; + + return 0; +} + +static av_cold void rgaoverlay_uninit(AVFilterContext *ctx) +{ + RGAOverlayContext *r = ctx->priv; + + ff_framesync_uninit(&r->fs); + + ff_rkrga_close(ctx); +} + +static int rgaoverlay_activate(AVFilterContext *ctx) +{ + RGAOverlayContext *r = ctx->priv; + AVFilterLink *inlink_main = ctx->inputs[0]; + AVFilterLink *inlink_overlay = ctx->inputs[1]; + AVFilterLink *outlink = ctx->outputs[0]; + int i, ret; + int64_t pts = AV_NOPTS_VALUE; + + ret = ff_framesync_activate(&r->fs); + if (ret < 0) + return ret; + + if (r->fs.eof) { + r->rga.eof = 1; + pts = r->fs.pts; + goto eof; + } + + if (r->rga.got_frame) + r->rga.got_frame = 0; + else { + for (i = 0; i < ctx->nb_inputs; i++) { + if (!ff_inlink_check_available_frame(ctx->inputs[i])) { + FF_FILTER_FORWARD_WANTED(outlink, ctx->inputs[i]); + } + } + return FFERROR_NOT_READY; + } + + return 0; + +eof: + ff_rkrga_filter_frame(&r->rga, + inlink_main, NULL, + inlink_overlay, NULL); + + pts = av_rescale_q(pts, inlink_main->time_base, outlink->time_base); + ff_outlink_set_status(outlink, AVERROR_EOF, pts); + return 0; +} + +#define OFFSET(x) offsetof(RGAOverlayContext, x) +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) + +static const AVOption rgaoverlay_options[] = { + { "x", "Overlay x position", OFFSET(overlay_ox), AV_OPT_TYPE_STRING, { .str = "0" }, 0, 0, .flags = FLAGS }, + { "y", "Overlay y position", OFFSET(overlay_oy), AV_OPT_TYPE_STRING, { .str = "0" }, 0, 0, .flags = FLAGS }, + { "alpha", "Overlay global alpha", OFFSET(global_alpha), AV_OPT_TYPE_INT, { .i64 = 255 }, 0, 255, .flags = FLAGS }, + { "alpha_format", "alpha format", OFFSET(alpha_format), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, .unit = "alpha_format" }, + { "straight", "The overlay input is unpremultiplied", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, .flags = FLAGS, .unit = "alpha_format" }, + { "premultiplied", "The overlay input is premultiplied", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, .flags = FLAGS, .unit = "alpha_format" }, + { "format", "Output video pixel format", OFFSET(format), AV_OPT_TYPE_PIXEL_FMT, { .i64 = AV_PIX_FMT_NONE }, INT_MIN, INT_MAX, .flags = FLAGS }, + { "eof_action", "Action to take when encountering EOF from secondary input ", + OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT }, + EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, .unit = "eof_action" }, + { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, .unit = "eof_action" }, + { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, .unit = "eof_action" }, + { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, .unit = "eof_action" }, + { "shortest", "Force termination when the shortest input terminates", OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, + { "repeatlast", "Repeat overlay of the last overlay frame", OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS }, + { "core", "Set multicore RGA scheduler core [use with caution]", OFFSET(rga.scheduler_core), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, INT_MAX, FLAGS, .unit = "core" }, + { "default", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, .unit = "core" }, + { "rga3_core0", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, .unit = "core" }, /* RGA3_SCHEDULER_CORE0 */ + { "rga3_core1", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, .unit = "core" }, /* RGA3_SCHEDULER_CORE1 */ + { "rga2_core0", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 4 }, 0, 0, FLAGS, .unit = "core" }, /* RGA2_SCHEDULER_CORE0 */ + { "rga2_core1", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 8 }, 0, 0, FLAGS, .unit = "core" }, /* RGA2_SCHEDULER_CORE1 */ + { "async_depth", "Set the internal parallelization depth", OFFSET(rga.async_depth), AV_OPT_TYPE_INT, { .i64 = 2 }, 0, 4, .flags = FLAGS }, + { "afbc", "Enable AFBC (Arm Frame Buffer Compression) to save bandwidth", OFFSET(rga.afbc_out), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, .flags = FLAGS }, + { NULL }, +}; + +FRAMESYNC_DEFINE_CLASS(rgaoverlay, RGAOverlayContext, fs); + +static const AVFilterPad rgaoverlay_inputs[] = { + { + .name = "main", + .type = AVMEDIA_TYPE_VIDEO, + }, + { + .name = "overlay", + .type = AVMEDIA_TYPE_VIDEO, + }, +}; + +static const AVFilterPad rgaoverlay_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = rgaoverlay_config_props, + }, +}; + +const AVFilter ff_vf_overlay_rkrga = { + .name = "overlay_rkrga", + .description = NULL_IF_CONFIG_SMALL("Rockchip RGA (2D Raster Graphic Acceleration) video compositor"), + .priv_size = sizeof(RGAOverlayContext), + .priv_class = &rgaoverlay_class, + .init = rgaoverlay_init, + .uninit = rgaoverlay_uninit, + .activate = rgaoverlay_activate, + FILTER_INPUTS(rgaoverlay_inputs), + FILTER_OUTPUTS(rgaoverlay_outputs), + FILTER_SINGLE_PIXFMT(AV_PIX_FMT_DRM_PRIME), + .preinit = rgaoverlay_framesync_preinit, + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, + .flags = AVFILTER_FLAG_HWDEVICE, +}; diff --git a/libavfilter/vf_vpp_rkrga.c b/libavfilter/vf_vpp_rkrga.c new file mode 100644 index 0000000000..e07247a251 --- /dev/null +++ b/libavfilter/vf_vpp_rkrga.c @@ -0,0 +1,581 @@ +/* + * Copyright (c) 2023 NyanMisaka + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Rockchip RGA (2D Raster Graphic Acceleration) video post-process (scale/crop/transpose) + */ + +#include "config_components.h" + +#include "libavutil/common.h" +#include "libavutil/eval.h" +#include "libavutil/internal.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" + +#include "filters.h" +#include "scale_eval.h" +#include "transpose.h" + +#include "rkrga_common.h" + +typedef struct RGAVppContext { + RKRGAContext rga; + + enum AVPixelFormat format; + int transpose; + int force_original_aspect_ratio; + int force_divisible_by; + int force_yuv; + int force_chroma; + int scheduler_core; + + int in_rotate_mode; + + char *ow, *oh; + char *cx, *cy, *cw, *ch; + int crop; + + int act_x, act_y; + int act_w, act_h; +} RGAVppContext; + +enum { + FORCE_YUV_DISABLE, + FORCE_YUV_AUTO, + FORCE_YUV_8BIT, + FORCE_YUV_10BIT, + FORCE_YUV_NB +}; + +enum { + FORCE_CHROMA_AUTO, + FORCE_CHROMA_420SP, + FORCE_CHROMA_420P, + FORCE_CHROMA_422SP, + FORCE_CHROMA_422P, + FORCE_CHROMA_NB +}; + +static const char *const var_names[] = { + "iw", "in_w", + "ih", "in_h", + "ow", "out_w", "w", + "oh", "out_h", "h", + "cw", + "ch", + "cx", + "cy", + "a", "dar", + "sar", + NULL +}; + +enum var_name { + VAR_IW, VAR_IN_W, + VAR_IH, VAR_IN_H, + VAR_OW, VAR_OUT_W, VAR_W, + VAR_OH, VAR_OUT_H, VAR_H, + VAR_CW, + VAR_CH, + VAR_CX, + VAR_CY, + VAR_A, VAR_DAR, + VAR_SAR, + VAR_VARS_NB +}; + +static av_cold int eval_expr(AVFilterContext *ctx, + int *ret_w, int *ret_h, + int *ret_cx, int *ret_cy, + int *ret_cw, int *ret_ch) +{ +#define PASS_EXPR(e, s) {\ + if (s) {\ + ret = av_expr_parse(&e, s, var_names, NULL, NULL, NULL, NULL, 0, ctx); \ + if (ret < 0) { \ + av_log(ctx, AV_LOG_ERROR, "Error when passing '%s'.\n", s); \ + goto release; \ + } \ + }\ +} +#define CALC_EXPR(e, v, i, d) {\ + if (e)\ + i = v = av_expr_eval(e, var_values, NULL); \ + else\ + i = v = d;\ +} + RGAVppContext *r = ctx->priv; + double var_values[VAR_VARS_NB] = { NAN }; + AVExpr *w_expr = NULL, *h_expr = NULL; + AVExpr *cw_expr = NULL, *ch_expr = NULL; + AVExpr *cx_expr = NULL, *cy_expr = NULL; + int ret = 0; + + PASS_EXPR(cw_expr, r->cw); + PASS_EXPR(ch_expr, r->ch); + + PASS_EXPR(w_expr, r->ow); + PASS_EXPR(h_expr, r->oh); + + PASS_EXPR(cx_expr, r->cx); + PASS_EXPR(cy_expr, r->cy); + + var_values[VAR_IW] = + var_values[VAR_IN_W] = ctx->inputs[0]->w; + + var_values[VAR_IH] = + var_values[VAR_IN_H] = ctx->inputs[0]->h; + + var_values[VAR_A] = (double)var_values[VAR_IN_W] / var_values[VAR_IN_H]; + var_values[VAR_SAR] = ctx->inputs[0]->sample_aspect_ratio.num ? + (double)ctx->inputs[0]->sample_aspect_ratio.num / ctx->inputs[0]->sample_aspect_ratio.den : 1; + var_values[VAR_DAR] = var_values[VAR_A] * var_values[VAR_SAR]; + + /* crop params */ + CALC_EXPR(cw_expr, var_values[VAR_CW], *ret_cw, var_values[VAR_IW]); + CALC_EXPR(ch_expr, var_values[VAR_CH], *ret_ch, var_values[VAR_IH]); + + /* calc again in case cw is relative to ch */ + CALC_EXPR(cw_expr, var_values[VAR_CW], *ret_cw, var_values[VAR_IW]); + + CALC_EXPR(w_expr, + var_values[VAR_OUT_W] = var_values[VAR_OW] = var_values[VAR_W], + *ret_w, var_values[VAR_CW]); + CALC_EXPR(h_expr, + var_values[VAR_OUT_H] = var_values[VAR_OH] = var_values[VAR_H], + *ret_h, var_values[VAR_CH]); + + /* calc again in case ow is relative to oh */ + CALC_EXPR(w_expr, + var_values[VAR_OUT_W] = var_values[VAR_OW] = var_values[VAR_W], + *ret_w, var_values[VAR_CW]); + + CALC_EXPR(cx_expr, var_values[VAR_CX], *ret_cx, (var_values[VAR_IW] - var_values[VAR_OW]) / 2); + CALC_EXPR(cy_expr, var_values[VAR_CY], *ret_cy, (var_values[VAR_IH] - var_values[VAR_OH]) / 2); + + /* calc again in case cx is relative to cy */ + CALC_EXPR(cx_expr, var_values[VAR_CX], *ret_cx, (var_values[VAR_IW] - var_values[VAR_OW]) / 2); + + r->crop = (*ret_cw != var_values[VAR_IW]) || (*ret_ch != var_values[VAR_IH]); + +release: + av_expr_free(w_expr); + av_expr_free(h_expr); + av_expr_free(cw_expr); + av_expr_free(ch_expr); + av_expr_free(cx_expr); + av_expr_free(cy_expr); +#undef PASS_EXPR +#undef CALC_EXPR + + return ret; +} + +static av_cold int set_size_info(AVFilterContext *ctx, + AVFilterLink *inlink, + AVFilterLink *outlink) +{ + RGAVppContext *r = ctx->priv; + int w, h, ret; + + if (inlink->w < 2 || inlink->w > 8192 || + inlink->h < 2 || inlink->h > 8192) { + av_log(ctx, AV_LOG_ERROR, "Supported input size is range from 2x2 ~ 8192x8192\n"); + return AVERROR(EINVAL); + } + + if ((ret = eval_expr(ctx, &w, &h, &r->act_x, &r->act_y, &r->act_w, &r->act_h)) < 0) + return ret; + + r->act_x = FFMAX(FFMIN(r->act_x, inlink->w), 0); + r->act_y = FFMAX(FFMIN(r->act_y, inlink->h), 0); + r->act_w = FFMAX(FFMIN(r->act_w, inlink->w), 0); + r->act_h = FFMAX(FFMIN(r->act_h, inlink->h), 0); + + r->act_x = FFMIN(r->act_x, inlink->w - r->act_w); + r->act_y = FFMIN(r->act_y, inlink->h - r->act_h); + r->act_w = FFMIN(r->act_w, inlink->w - r->act_x); + r->act_h = FFMIN(r->act_h, inlink->h - r->act_y); + + ff_scale_adjust_dimensions(inlink, &w, &h, + r->force_original_aspect_ratio, r->force_divisible_by); + + if (((int64_t)h * inlink->w) > INT_MAX || + ((int64_t)w * inlink->h) > INT_MAX) { + av_log(ctx, AV_LOG_ERROR, "Rescaled value for width or height is too big.\n"); + return AVERROR(EINVAL); + } + + outlink->w = w; + outlink->h = h; + if (outlink->w < 2 || outlink->w > 8192 || + outlink->h < 2 || outlink->h > 8192) { + av_log(ctx, AV_LOG_ERROR, "Supported output size is range from 2x2 ~ 8192x8192\n"); + return AVERROR(EINVAL); + } + + if (inlink->sample_aspect_ratio.num) + outlink->sample_aspect_ratio = av_mul_q((AVRational){outlink->h * inlink->w, + outlink->w * inlink->h}, + inlink->sample_aspect_ratio); + else + outlink->sample_aspect_ratio = inlink->sample_aspect_ratio; + + if (r->transpose >= 0) { + switch (r->transpose) { + case TRANSPOSE_CCLOCK_FLIP: + r->in_rotate_mode = 0x07 | (0x01 << 4); /* HAL_TRANSFORM_ROT_270 | (HAL_TRANSFORM_FLIP_H << 4) */ + FFSWAP(int, outlink->w, outlink->h); + FFSWAP(int, outlink->sample_aspect_ratio.num, outlink->sample_aspect_ratio.den); + break; + case TRANSPOSE_CLOCK: + r->in_rotate_mode = 0x04; /* HAL_TRANSFORM_ROT_90 */ + FFSWAP(int, outlink->w, outlink->h); + FFSWAP(int, outlink->sample_aspect_ratio.num, outlink->sample_aspect_ratio.den); + break; + case TRANSPOSE_CCLOCK: + r->in_rotate_mode = 0x07; /* HAL_TRANSFORM_ROT_270 */ + FFSWAP(int, outlink->w, outlink->h); + FFSWAP(int, outlink->sample_aspect_ratio.num, outlink->sample_aspect_ratio.den); + break; + case TRANSPOSE_CLOCK_FLIP: + r->in_rotate_mode = 0x04 | (0x01 << 4); /* HAL_TRANSFORM_ROT_90 | (HAL_TRANSFORM_FLIP_H << 4) */ + FFSWAP(int, outlink->w, outlink->h); + FFSWAP(int, outlink->sample_aspect_ratio.num, outlink->sample_aspect_ratio.den); + break; + case TRANSPOSE_REVERSAL: + r->in_rotate_mode = 0x03; /* HAL_TRANSFORM_ROT_180 */ + break; + case TRANSPOSE_HFLIP: + r->in_rotate_mode = 0x01; /* HAL_TRANSFORM_FLIP_H */ + break; + case TRANSPOSE_VFLIP: + r->in_rotate_mode = 0x02; /* HAL_TRANSFORM_FLIP_V */ + break; + default: + av_log(ctx, AV_LOG_ERROR, "Failed to set transpose mode to %d\n", r->transpose); + return AVERROR(EINVAL); + } + } + + return 0; +} + +static av_cold void config_force_format(AVFilterContext *ctx, + enum AVPixelFormat in_format, + enum AVPixelFormat *out_format) +{ + RGAVppContext *r = ctx->priv; + const AVPixFmtDescriptor *desc; + const char *rga_ver = NULL; + int has_rga3 = 0; + int out_depth, force_chroma; + int is_yuv, is_fully_planar; + + if (!out_format) + return; + + if (r->force_yuv == FORCE_YUV_AUTO) + out_depth = (in_format == AV_PIX_FMT_NV15 || + in_format == AV_PIX_FMT_NV20) ? 10 : 0; + else + out_depth = (r->force_yuv == FORCE_YUV_8BIT) ? 8 : + (r->force_yuv == FORCE_YUV_10BIT) ? 10 : 0; + + if (!out_depth) + return; + + /* Auto fallback to 8-bit fmts on RGA2 */ + rga_ver = querystring(RGA_VERSION); + has_rga3 = !!strstr(rga_ver, "RGA_3"); + if (out_depth >= 10 && !has_rga3) + out_depth = 8; + + desc = av_pix_fmt_desc_get(in_format); + is_yuv = !(desc->flags & AV_PIX_FMT_FLAG_RGB) && desc->nb_components >= 2; + + force_chroma = r->force_chroma; + if (is_yuv && force_chroma == FORCE_CHROMA_AUTO) { + is_fully_planar = (desc->flags & AV_PIX_FMT_FLAG_PLANAR) && + desc->comp[1].plane != desc->comp[2].plane; + if (desc->log2_chroma_w == 1 && desc->log2_chroma_h == 1) + force_chroma = is_fully_planar ? FORCE_CHROMA_420P : FORCE_CHROMA_420SP; + else if (desc->log2_chroma_w == 1 && !desc->log2_chroma_h) + force_chroma = is_fully_planar ? FORCE_CHROMA_422P : FORCE_CHROMA_422SP; + } + + switch (force_chroma) { + case FORCE_CHROMA_422P: + *out_format = AV_PIX_FMT_YUV422P; + break; + case FORCE_CHROMA_422SP: + *out_format = out_depth == 10 ? + AV_PIX_FMT_P210 : AV_PIX_FMT_NV16; + break; + case FORCE_CHROMA_420P: + *out_format = AV_PIX_FMT_YUV420P; + break; + case FORCE_CHROMA_420SP: + default: + *out_format = out_depth == 10 ? + AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; + } +} + +static av_cold int rgavpp_config_props(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + RGAVppContext *r = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVHWFramesContext *in_frames_ctx; + enum AVPixelFormat in_format; + enum AVPixelFormat out_format; + RKRGAParam param = { NULL }; + int ret; + + if (!inlink->hw_frames_ctx) { + av_log(ctx, AV_LOG_ERROR, "No hw context provided on input\n"); + return AVERROR(EINVAL); + } + in_frames_ctx = (AVHWFramesContext *)inlink->hw_frames_ctx->data; + in_format = in_frames_ctx->sw_format; + out_format = (r->format == AV_PIX_FMT_NONE) ? in_format : r->format; + + config_force_format(ctx, in_format, &out_format); + + ret = set_size_info(ctx, inlink, outlink); + if (ret < 0) + return ret; + + param.filter_frame = NULL; + param.out_sw_format = out_format; + param.in_rotate_mode = r->in_rotate_mode; + param.in_crop = r->crop; + param.in_crop_x = r->act_x; + param.in_crop_y = r->act_y; + param.in_crop_w = r->act_w; + param.in_crop_h = r->act_h; + + ret = ff_rkrga_init(ctx, ¶m); + if (ret < 0) + return ret; + + av_log(ctx, AV_LOG_VERBOSE, "w:%d h:%d fmt:%s -> w:%d h:%d fmt:%s\n", + inlink->w, inlink->h, av_get_pix_fmt_name(in_format), + outlink->w, outlink->h, av_get_pix_fmt_name(out_format)); + + return 0; +} + +static int rgavpp_activate(AVFilterContext *ctx) +{ + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *outlink = ctx->outputs[0]; + RGAVppContext *r = ctx->priv; + AVFrame *in = NULL; + int ret, at_eof = 0, status = 0; + int64_t pts = AV_NOPTS_VALUE; + + FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); + + if (r->rga.eof) + at_eof = 1; + else { + ret = ff_inlink_consume_frame(inlink, &in); + if (ret < 0) + return ret; + + if (ff_inlink_acknowledge_status(inlink, &status, &pts)) { + if (status == AVERROR_EOF) { + at_eof = 1; + } + } + } + + if (in) { + ret = ff_rkrga_filter_frame(&r->rga, inlink, in, NULL, NULL); + av_frame_free(&in); + if (ret < 0) + return ret; + else if (!r->rga.got_frame) + goto not_ready; + + if (at_eof) { + r->rga.eof = 1; + goto eof; + } + + if (r->rga.got_frame) { + r->rga.got_frame = 0; + return 0; + } + } + +not_ready: + if (at_eof) { + r->rga.eof = 1; + goto eof; + } + + FF_FILTER_FORWARD_WANTED(outlink, inlink); + return FFERROR_NOT_READY; + +eof: + ff_rkrga_filter_frame(&r->rga, inlink, NULL, NULL, NULL); + + pts = av_rescale_q(pts, inlink->time_base, outlink->time_base); + ff_outlink_set_status(outlink, AVERROR_EOF, pts); + return 0; +} + +static av_cold int rgavpp_init(AVFilterContext *ctx) +{ + return 0; +} + +static av_cold void rgavpp_uninit(AVFilterContext *ctx) +{ + ff_rkrga_close(ctx); +} + +#define OFFSET(x) offsetof(RGAVppContext, x) +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) + +#define RKRGA_VPP_COMMON_OPTS \ + { "force_yuv", "Enforce planar YUV format output", OFFSET(force_yuv), AV_OPT_TYPE_INT, { .i64 = FORCE_YUV_DISABLE }, 0, FORCE_YUV_NB - 1, FLAGS, .unit = "force_yuv" }, \ + { "disable", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_YUV_DISABLE }, 0, 0, FLAGS, .unit = "force_yuv" }, \ + { "auto", "Match in/out bit depth", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_YUV_AUTO }, 0, 0, FLAGS, .unit = "force_yuv" }, \ + { "8bit", "8-bit", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_YUV_8BIT }, 0, 0, FLAGS, .unit = "force_yuv" }, \ + { "10bit", "10-bit uncompact/8-bit", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_YUV_10BIT }, 0, 0, FLAGS, .unit = "force_yuv" }, \ + { "force_chroma", "Enforce chroma of planar YUV format output", OFFSET(force_chroma), AV_OPT_TYPE_INT, { .i64 = FORCE_CHROMA_AUTO }, 0, FORCE_CHROMA_NB - 1, FLAGS, .unit = "force_chroma" }, \ + { "auto", "Match in/out chroma", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_CHROMA_AUTO }, 0, 0, FLAGS, .unit = "force_chroma" }, \ + { "420sp", "4:2:0 semi-planar", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_CHROMA_420SP }, 0, 0, FLAGS, .unit = "force_chroma" }, \ + { "420p", "4:2:0 fully-planar", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_CHROMA_420P }, 0, 0, FLAGS, .unit = "force_chroma" }, \ + { "422sp", "4:2:2 semi-planar", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_CHROMA_422SP }, 0, 0, FLAGS, .unit = "force_chroma" }, \ + { "422p", "4:2:2 fully-planar", 0, AV_OPT_TYPE_CONST, { .i64 = FORCE_CHROMA_422P }, 0, 0, FLAGS, .unit = "force_chroma" }, \ + { "core", "Set multicore RGA scheduler core [use with caution]", OFFSET(rga.scheduler_core), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, INT_MAX, FLAGS, .unit = "core" }, \ + { "default", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, .unit = "core" }, \ + { "rga3_core0", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, .unit = "core" }, /* RGA3_SCHEDULER_CORE0 */ \ + { "rga3_core1", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, .unit = "core" }, /* RGA3_SCHEDULER_CORE1 */ \ + { "rga2_core0", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 4 }, 0, 0, FLAGS, .unit = "core" }, /* RGA2_SCHEDULER_CORE0 */ \ + { "rga2_core1", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 8 }, 0, 0, FLAGS, .unit = "core" }, /* RGA2_SCHEDULER_CORE1 */ \ + { "async_depth", "Set the internal parallelization depth", OFFSET(rga.async_depth), AV_OPT_TYPE_INT, { .i64 = 2 }, 0, 4, .flags = FLAGS }, \ + { "afbc", "Enable AFBC (Arm Frame Buffer Compression) to save bandwidth", OFFSET(rga.afbc_out), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, .flags = FLAGS }, + +static const AVFilterPad rgavpp_inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + }, +}; + +static const AVFilterPad rgavpp_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = rgavpp_config_props, + }, +}; + +#if CONFIG_SCALE_RKRGA_FILTER + +static const AVOption rgascale_options[] = { + { "w", "Output video width", OFFSET(ow), AV_OPT_TYPE_STRING, { .str = "iw" }, 0, 0, FLAGS }, + { "h", "Output video height", OFFSET(oh), AV_OPT_TYPE_STRING, { .str = "ih" }, 0, 0, FLAGS }, + { "format", "Output video pixel format", OFFSET(format), AV_OPT_TYPE_PIXEL_FMT, { .i64 = AV_PIX_FMT_NONE }, INT_MIN, INT_MAX, .flags = FLAGS }, + { "force_original_aspect_ratio", "Decrease or increase w/h if necessary to keep the original AR", OFFSET(force_original_aspect_ratio), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 2, FLAGS, .unit = "force_oar" }, \ + { "disable", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, .unit = "force_oar" }, \ + { "decrease", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, .unit = "force_oar" }, \ + { "increase", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, .unit = "force_oar" }, \ + { "force_divisible_by", "Enforce that the output resolution is divisible by a defined integer when force_original_aspect_ratio is used", OFFSET(force_divisible_by), AV_OPT_TYPE_INT, { .i64 = 2 }, 1, 256, FLAGS }, \ + RKRGA_VPP_COMMON_OPTS + { NULL }, +}; + +static av_cold int rgascale_preinit(AVFilterContext *ctx) +{ + RGAVppContext *r = ctx->priv; + + r->transpose = -1; + return 0; +} + +AVFILTER_DEFINE_CLASS(rgascale); + +const AVFilter ff_vf_scale_rkrga = { + .name = "scale_rkrga", + .description = NULL_IF_CONFIG_SMALL("Rockchip RGA (2D Raster Graphic Acceleration) video resizer and format converter"), + .priv_size = sizeof(RGAVppContext), + .priv_class = &rgascale_class, + .preinit = rgascale_preinit, + .init = rgavpp_init, + .uninit = rgavpp_uninit, + FILTER_INPUTS(rgavpp_inputs), + FILTER_OUTPUTS(rgavpp_outputs), + FILTER_SINGLE_PIXFMT(AV_PIX_FMT_DRM_PRIME), + .activate = rgavpp_activate, + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, + .flags = AVFILTER_FLAG_HWDEVICE, +}; + +#endif + +#if CONFIG_VPP_RKRGA_FILTER + +static const AVOption rgavpp_options[] = { + { "w", "Output video width", OFFSET(ow), AV_OPT_TYPE_STRING, { .str = "cw" }, 0, 0, FLAGS }, + { "h", "Output video height", OFFSET(oh), AV_OPT_TYPE_STRING, { .str = "w*ch/cw" }, 0, 0, FLAGS }, + { "cw", "Set the width crop area expression", OFFSET(cw), AV_OPT_TYPE_STRING, { .str = "iw" }, 0, 0, FLAGS }, + { "ch", "Set the height crop area expression", OFFSET(ch), AV_OPT_TYPE_STRING, { .str = "ih" }, 0, 0, FLAGS }, + { "cx", "Set the x crop area expression", OFFSET(cx), AV_OPT_TYPE_STRING, { .str = "(in_w-out_w)/2" }, 0, 0, FLAGS }, + { "cy", "Set the y crop area expression", OFFSET(cy), AV_OPT_TYPE_STRING, { .str = "(in_h-out_h)/2" }, 0, 0, FLAGS }, + { "format", "Output video pixel format", OFFSET(format), AV_OPT_TYPE_PIXEL_FMT, { .i64 = AV_PIX_FMT_NONE }, INT_MIN, INT_MAX, .flags = FLAGS }, + { "transpose", "Set transpose direction", OFFSET(transpose), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, 6, FLAGS, .unit = "transpose" }, + { "cclock_hflip", "Rotate counter-clockwise with horizontal flip", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CCLOCK_FLIP }, 0, 0, FLAGS, .unit = "transpose" }, + { "clock", "Rotate clockwise", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CLOCK }, 0, 0, FLAGS, .unit = "transpose" }, + { "cclock", "Rotate counter-clockwise", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CCLOCK }, 0, 0, FLAGS, .unit = "transpose" }, + { "clock_hflip", "Rotate clockwise with horizontal flip", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_CLOCK_FLIP }, 0, 0, FLAGS, .unit = "transpose" }, + { "reversal", "Rotate by half-turn", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_REVERSAL }, 0, 0, FLAGS, .unit = "transpose" }, + { "hflip", "Flip horizontally", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_HFLIP }, 0, 0, FLAGS, .unit = "transpose" }, + { "vflip", "Flip vertically", 0, AV_OPT_TYPE_CONST, { .i64 = TRANSPOSE_VFLIP }, 0, 0, FLAGS, .unit = "transpose" }, + RKRGA_VPP_COMMON_OPTS + { NULL }, +}; + +AVFILTER_DEFINE_CLASS(rgavpp); + +const AVFilter ff_vf_vpp_rkrga = { + .name = "vpp_rkrga", + .description = NULL_IF_CONFIG_SMALL("Rockchip RGA (2D Raster Graphic Acceleration) video post-process (scale/crop/transpose)"), + .priv_size = sizeof(RGAVppContext), + .priv_class = &rgavpp_class, + .init = rgavpp_init, + .uninit = rgavpp_uninit, + FILTER_INPUTS(rgavpp_inputs), + FILTER_OUTPUTS(rgavpp_outputs), + FILTER_SINGLE_PIXFMT(AV_PIX_FMT_DRM_PRIME), + .activate = rgavpp_activate, + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, + .flags = AVFILTER_FLAG_HWDEVICE, +}; + +#endif