Add MBtree support for libx264

Patch by Erik Slagter

Originally committed as revision 20774 to svn://svn.ffmpeg.org/ffmpeg/trunk

ffpresets/libx264-baseline.ffpreset

 coder=0
 bf=0
-flags2=-wpred-dct8x8
+flags2=-wpred-dct8x8+mbtree
 wpredp=0

ffpresets/libx264-default.ffpreset

@@ -18,5 +18,5 @@ bf=3
 refs=3
 directpred=1
 trellis=1
-flags2=+mixed_refs+wpred+dct8x8+fastpskip
+flags2=+mixed_refs+wpred+dct8x8+fastpskip+mbtree
 wpredp=2

ffpresets/libx264-fastfirstpass.ffpreset

@@ -18,5 +18,5 @@ bf=3
 refs=1
 directpred=3
 trellis=0
-flags2=-bpyramid-wpred-mixed_refs-dct8x8+fastpskip
+flags2=-bpyramid-wpred-mixed_refs-dct8x8+fastpskip+mbtree
 wpredp=2

ffpresets/libx264-hq.ffpreset

@@ -18,5 +18,5 @@ bf=3
 refs=4
 directpred=3
 trellis=1
-flags2=+wpred+mixed_refs+dct8x8+fastpskip
+flags2=+wpred+mixed_refs+dct8x8+fastpskip+mbtree
 wpredp=2

ffpresets/libx264-ipod320.ffpreset

 coder=0
 bf=0
-flags2=-wpred-dct8x8
+flags2=-wpred-dct8x8+mbtree
 level=13
 maxrate=768000
 bufsize=3000000

ffpresets/libx264-ipod640.ffpreset

 coder=0
 bf=0
 refs=1
-flags2=-wpred-dct8x8
+flags2=-wpred-dct8x8+mbtree
 level=30
 maxrate=10000000
 bufsize=10000000

ffpresets/libx264-lossless_fast.ffpreset

@@ -15,6 +15,6 @@ qmin=10
 qmax=51
 qdiff=4
 directpred=1
-flags2=+fastpskip
+flags2=+fastpskip+mbtree
 cqp=0
 wpredp=0

ffpresets/libx264-lossless_max.ffpreset

@@ -16,6 +16,6 @@ qmax=51
 qdiff=4
 refs=16
 directpred=1
-flags2=+mixed_refs+dct8x8+fastpskip
+flags2=+mixed_refs+dct8x8+fastpskip+mbtree
 cqp=0
 wpredp=2

ffpresets/libx264-lossless_medium.ffpreset

@@ -15,6 +15,6 @@ qmin=10
 qmax=51
 qdiff=4
 directpred=1
-flags2=+fastpskip
+flags2=+fastpskip+mbtree
 cqp=0
 wpredp=2

ffpresets/libx264-lossless_slow.ffpreset

@@ -16,6 +16,6 @@ qmax=51
 qdiff=4
 refs=2
 directpred=1
-flags2=+dct8x8+fastpskip
+flags2=+dct8x8+fastpskip+mbtree
 cqp=0
 wpredp=2

ffpresets/libx264-lossless_slower.ffpreset

@@ -16,6 +16,6 @@ qmax=51
 qdiff=4
 refs=4
 directpred=1
-flags2=+mixed_refs+dct8x8+fastpskip
+flags2=+mixed_refs+dct8x8+fastpskip+mbtree
 cqp=0
 wpredp=2

ffpresets/libx264-lossless_ultrafast.ffpreset

@@ -15,5 +15,5 @@ qmin=10
 qmax=51
 qdiff=4
 directpred=1
-flags2=+fastpskip
+flags2=+fastpskip+mbtree
 cqp=0

ffpresets/libx264-main.ffpreset

-flags2=-dct8x8
+flags2=-dct8x8+mbtree

ffpresets/libx264-max.ffpreset

@@ -18,5 +18,5 @@ bf=3
 refs=16
 directpred=3
 trellis=2
-flags2=+wpred+mixed_refs+dct8x8-fastpskip
+flags2=+wpred+mixed_refs+dct8x8-fastpskip+mbtree
 wpredp=2

ffpresets/libx264-normal.ffpreset

@@ -18,5 +18,5 @@ bf=3
 refs=2
 directpred=3
 trellis=0
-flags2=+wpred+dct8x8+fastpskip
+flags2=+wpred+dct8x8+fastpskip+mbtree
 wpredp=2

ffpresets/libx264-slowfirstpass.ffpreset

@@ -18,5 +18,5 @@ bf=3
 refs=1
 directpred=3
 trellis=0
-flags2=+wpred+dct8x8+fastpskip
+flags2=+wpred+dct8x8+fastpskip+mbtree
 wpredp=2

libavcodec/avcodec.h

@@ -580,6 +580,7 @@ typedef struct RcOverride{
 #define CODEC_FLAG2_CHUNKS        0x00008000 ///< Input bitstream might be truncated at a packet boundaries instead of only at frame boundaries.
 #define CODEC_FLAG2_NON_LINEAR_QUANT 0x00010000 ///< Use MPEG-2 nonlinear quantizer.
 #define CODEC_FLAG2_BIT_RESERVOIR 0x00020000 ///< Use a bit reservoir when encoding if possible
+#define CODEC_FLAG2_MBTREE        0x00040000 ///< Use macroblock tree ratecontrol (x264 only)
 
 /* Unsupported options :
  *              Syntax Arithmetic coding (SAC)

libavcodec/libx264.c

@@ -266,6 +266,7 @@ static av_cold int X264_init(AVCodecContext *avctx)
     } else
         x4->params.rc.f_vbv_buffer_init = 0.9;
 
+    x4->params.rc.b_mb_tree               = !!(avctx->flags2 & CODEC_FLAG2_MBTREE);
     x4->params.rc.f_ip_factor             = 1 / fabs(avctx->i_quant_factor);
     x4->params.rc.f_pb_factor             = avctx->b_quant_factor;
     x4->params.analyse.i_chroma_qp_offset = avctx->chromaoffset;

libavcodec/options.c

@@ -391,6 +391,7 @@ static const AVOption options[]={
 {"request_channels", "set desired number of audio channels", OFFSET(request_channels), FF_OPT_TYPE_INT, DEFAULT, 0, INT_MAX, A|D},
 {"drc_scale", "percentage of dynamic range compression to apply", OFFSET(drc_scale), FF_OPT_TYPE_FLOAT, 1.0, 0.0, 1.0, A|D},
 {"reservoir", "use bit reservoir", 0, FF_OPT_TYPE_CONST, CODEC_FLAG2_BIT_RESERVOIR, INT_MIN, INT_MAX, A|E, "flags2"},
+{"mbtree", "use macroblock tree ratecontrol (x264 only)", 0, FF_OPT_TYPE_CONST, CODEC_FLAG2_MBTREE, INT_MIN, INT_MAX, V|E, "flags2"},
 {"bits_per_raw_sample", NULL, OFFSET(bits_per_raw_sample), FF_OPT_TYPE_INT, DEFAULT, INT_MIN, INT_MAX},
 {"channel_layout", NULL, OFFSET(channel_layout), FF_OPT_TYPE_INT64, DEFAULT, 0, INT64_MAX, A|E|D, "channel_layout"},
 {"request_channel_layout", NULL, OFFSET(request_channel_layout), FF_OPT_TYPE_INT64, DEFAULT, 0, INT64_MAX, A|D, "request_channel_layout"},