libx265: Update ROI behaviour to match documentation

Equivalent to the previous patch for libx264.

libavcodec/libx265.c

@@ -316,27 +316,36 @@ static av_cold int libx265_encode_set_roi(libx265Context *ctx, const AVFrame *fr
             int mb_size = (ctx->params->rc.qgSize == 8) ? 8 : 16;
             int mbx = (frame->width + mb_size - 1) / mb_size;
             int mby = (frame->height + mb_size - 1) / mb_size;
+            int qp_range = 51 + 6 * (pic->bitDepth - 8);
             int nb_rois;
-            AVRegionOfInterest *roi;
+            const AVRegionOfInterest *roi;
+            uint32_t roi_size;
             float *qoffsets;         /* will be freed after encode is called. */
+
+            roi = (const AVRegionOfInterest*)sd->data;
+            roi_size = roi->self_size;
+            if (!roi_size || sd->size % roi_size != 0) {
+                av_log(ctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");
+                return AVERROR(EINVAL);
+            }
+            nb_rois = sd->size / roi_size;
+
             qoffsets = av_mallocz_array(mbx * mby, sizeof(*qoffsets));
             if (!qoffsets)
                 return AVERROR(ENOMEM);
 
-            nb_rois = sd->size / sizeof(AVRegionOfInterest);
-            roi = (AVRegionOfInterest*)sd->data;
-            for (int count = 0; count < nb_rois; count++) {
-                int starty = FFMIN(mby, roi->top / mb_size);
-                int endy   = FFMIN(mby, (roi->bottom + mb_size - 1)/ mb_size);
-                int startx = FFMIN(mbx, roi->left / mb_size);
-                int endx   = FFMIN(mbx, (roi->right + mb_size - 1)/ mb_size);
+            // This list must be iterated in reverse because the first
+            // region in the list applies when regions overlap.
+            for (int i = nb_rois - 1; i >= 0; i--) {
+                int startx, endx, starty, endy;
                 float qoffset;
 
-                if (roi->self_size == 0) {
-                    av_free(qoffsets);
-                    av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.self_size must be set to sizeof(AVRegionOfInterest).\n");
-                    return AVERROR(EINVAL);
-                }
+                roi = (const AVRegionOfInterest*)(sd->data + roi_size * i);
+
+                starty = FFMIN(mby, roi->top / mb_size);
+                endy   = FFMIN(mby, (roi->bottom + mb_size - 1)/ mb_size);
+                startx = FFMIN(mbx, roi->left / mb_size);
+                endx   = FFMIN(mbx, (roi->right + mb_size - 1)/ mb_size);
 
                 if (roi->qoffset.den == 0) {
                     av_free(qoffsets);
@@ -344,18 +353,11 @@ static av_cold int libx265_encode_set_roi(libx265Context *ctx, const AVFrame *fr
                     return AVERROR(EINVAL);
                 }
                 qoffset = roi->qoffset.num * 1.0f / roi->qoffset.den;
-                qoffset = av_clipf(qoffset, -1.0f, 1.0f);
-
-                /* qp range of x265 is from 0 to 51, just choose 25 as the scale value,
-                 * so the range of final qoffset is [-25.0, 25.0].
-                 */
-                qoffset = qoffset * 25;
+                qoffset = av_clipf(qoffset * qp_range, -qp_range, +qp_range);
 
                 for (int y = starty; y < endy; y++)
                     for (int x = startx; x < endx; x++)
                         qoffsets[x + y*mbx] = qoffset;
-
-                roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);
             }
 
             pic->quantOffsets = qoffsets;