libavfilter: add transpose_vaapi filter

Swap width and height when do clock/cclock rotation
Add reversal/hflip/vflip options

ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128
-hwaccel_output_format vaapi -i input.264 -vf "transpose_vaapi=clock_flip"
-c:v h264_vaapi output.h264
Signed-off-by: Zachary Zhou <zachary.zhou@intel.com>
Signed-off-by: Mark Thompson <sw@jkqxz.net>

configure

@@ -3483,6 +3483,7 @@ tinterlace_pad_test_deps="tinterlace_filter"
 tonemap_filter_deps="const_nan"
 tonemap_opencl_filter_deps="opencl const_nan"
 transpose_opencl_filter_deps="opencl"
+transpose_vaapi_filter_deps="vaapi VAProcPipelineCaps_rotation_flags"
 unsharp_opencl_filter_deps="opencl"
 uspp_filter_deps="gpl avcodec"
 vaguedenoiser_filter_deps="gpl"
@@ -5986,6 +5987,7 @@ check_type "d3d9.h dxva2api.h" DXVA2_ConfigPictureDecode -D_WIN32_WINNT=0x0602
 
 check_type "va/va.h va/va_dec_hevc.h" "VAPictureParameterBufferHEVC"
 check_struct "va/va.h" "VADecPictureParameterBufferVP9" bit_depth
+check_struct "va/va.h va/va_vpp.h" "VAProcPipelineCaps" rotation_flags
 check_type "va/va.h va/va_enc_hevc.h" "VAEncPictureParameterBufferHEVC"
 check_type "va/va.h va/va_enc_jpeg.h" "VAEncPictureParameterBufferJPEG"
 check_type "va/va.h va/va_enc_vp8.h"  "VAEncPictureParameterBufferVP8"

libavfilter/Makefile

@@ -396,6 +396,7 @@ OBJS-$(CONFIG_TPAD_FILTER)                   += vf_tpad.o
 OBJS-$(CONFIG_TRANSPOSE_FILTER)              += vf_transpose.o
 OBJS-$(CONFIG_TRANSPOSE_NPP_FILTER)          += vf_transpose_npp.o cuda_check.o
 OBJS-$(CONFIG_TRANSPOSE_OPENCL_FILTER)       += vf_transpose_opencl.o opencl.o opencl/transpose.o
+OBJS-$(CONFIG_TRANSPOSE_VAAPI_FILTER)        += vf_transpose_vaapi.o vaapi_vpp.o
 OBJS-$(CONFIG_TRIM_FILTER)                   += trim.o
 OBJS-$(CONFIG_UNPREMULTIPLY_FILTER)          += vf_premultiply.o framesync.o
 OBJS-$(CONFIG_UNSHARP_FILTER)                += vf_unsharp.o

libavfilter/allfilters.c

@@ -375,6 +375,7 @@ extern AVFilter ff_vf_tpad;
 extern AVFilter ff_vf_transpose;
 extern AVFilter ff_vf_transpose_npp;
 extern AVFilter ff_vf_transpose_opencl;
+extern AVFilter ff_vf_transpose_vaapi;
 extern AVFilter ff_vf_trim;
 extern AVFilter ff_vf_unpremultiply;
 extern AVFilter ff_vf_unsharp;

libavfilter/transpose.h

@@ -29,6 +29,9 @@ enum TransposeDir {
     TRANSPOSE_CLOCK,
     TRANSPOSE_CCLOCK,
     TRANSPOSE_CLOCK_FLIP,
+    TRANSPOSE_REVERSAL,    // rotate by half-turn
+    TRANSPOSE_HFLIP,
+    TRANSPOSE_VFLIP,
 };
 
 #endif