avcodec/libx265: add support for ROI-based encoding

Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>

libavcodec/libx265.c

@@ -285,6 +285,65 @@ static av_cold int libx265_encode_init(AVCodecContext *avctx)
     return 0;
 }
 
+static av_cold int libx265_encode_set_roi(libx265Context *ctx, const AVFrame *frame, x265_picture* pic)
+{
+    AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
+    if (sd) {
+        if (ctx->params->rc.aqMode == X265_AQ_NONE) {
+            av_log(ctx, AV_LOG_WARNING, "Adaptive quantization must be enabled to use ROI encoding, skipping ROI.\n");
+        } else {
+            /* 8x8 block when qg-size is 8, 16*16 block otherwise. */
+            int mb_size = (ctx->params->rc.qgSize == 8) ? 8 : 16;
+            int mbx = (frame->width + mb_size - 1) / mb_size;
+            int mby = (frame->height + mb_size - 1) / mb_size;
+            int nb_rois;
+            AVRegionOfInterest *roi;
+            float *qoffsets;         /* will be freed after encode is called. */
+            qoffsets = av_mallocz_array(mbx * mby, sizeof(*qoffsets));
+            if (!qoffsets)
+                return AVERROR(ENOMEM);
+
+            nb_rois = sd->size / sizeof(AVRegionOfInterest);
+            roi = (AVRegionOfInterest*)sd->data;
+            for (int count = 0; count < nb_rois; count++) {
+                int starty = FFMIN(mby, roi->top / mb_size);
+                int endy   = FFMIN(mby, (roi->bottom + mb_size - 1)/ mb_size);
+                int startx = FFMIN(mbx, roi->left / mb_size);
+                int endx   = FFMIN(mbx, (roi->right + mb_size - 1)/ mb_size);
+                float qoffset;
+
+                if (roi->self_size == 0) {
+                    av_free(qoffsets);
+                    av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.self_size must be set to sizeof(AVRegionOfInterest).\n");
+                    return AVERROR(EINVAL);
+                }
+
+                if (roi->qoffset.den == 0) {
+                    av_free(qoffsets);
+                    av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n");
+                    return AVERROR(EINVAL);
+                }
+                qoffset = roi->qoffset.num * 1.0f / roi->qoffset.den;
+                qoffset = av_clipf(qoffset, -1.0f, 1.0f);
+
+                /* qp range of x265 is from 0 to 51, just choose 25 as the scale value,
+                 * so the range of final qoffset is [-25.0, 25.0].
+                 */
+                qoffset = qoffset * 25;
+
+                for (int y = starty; y < endy; y++)
+                    for (int x = startx; x < endx; x++)
+                        qoffsets[x + y*mbx] = qoffset;
+
+                roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);
+            }
+
+            pic->quantOffsets = qoffsets;
+        }
+    }
+    return 0;
+}
+
 static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                                 const AVFrame *pic, int *got_packet)
 {
@@ -314,10 +373,17 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                             pic->pict_type == AV_PICTURE_TYPE_P ? X265_TYPE_P :
                             pic->pict_type == AV_PICTURE_TYPE_B ? X265_TYPE_B :
                             X265_TYPE_AUTO;
+
+        ret = libx265_encode_set_roi(ctx, pic, &x265pic);
+        if (ret < 0)
+            return ret;
     }
 
     ret = ctx->api->encoder_encode(ctx->encoder, &nal, &nnal,
                                    pic ? &x265pic : NULL, &x265pic_out);
+
+    av_freep(&x265pic.quantOffsets);
+
     if (ret < 0)
         return AVERROR_EXTERNAL;