Merge commit '7b1fbd47'

* commit '7b1fbd47':
  indeo2: check decoding errors.
  indeo2: return meaningful error codes
  rl2: cosmetics, reformat

Conflicts:
	libavcodec/rl2.c
Merged-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/indeo2.c

@@ -57,7 +57,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst
     int t;
 
     if(width&1)
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     /* first line contain absolute values, other lines contain deltas */
     while (out < width){
@@ -65,7 +65,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst
         if(c >= 0x80) { /* we have a run */
             c -= 0x7F;
             if(out + c*2 > width)
-                return -1;
+                return AVERROR_INVALIDDATA;
             for (i = 0; i < c * 2; i++)
                 dst[out++] = 0x80;
         } else { /* copy two values from table */
@@ -82,7 +82,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst
             if(c >= 0x80) { /* we have a skip */
                 c -= 0x7F;
                 if(out + c*2 > width)
-                    return -1;
+                    return AVERROR_INVALIDDATA;
                 for (i = 0; i < c * 2; i++) {
                     dst[out] = dst[out - stride];
                     out++;
@@ -112,7 +112,7 @@ static int ir2_decode_plane_inter(Ir2Context *ctx, int width, int height, uint8_
     int t;
 
     if(width&1)
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     for (j = 0; j < height; j++){
         out = 0;
@@ -146,13 +146,13 @@ static int ir2_decode_frame(AVCodecContext *avctx,
     Ir2Context * const s = avctx->priv_data;
     AVFrame *picture = data;
     AVFrame * const p = &s->picture;
-    int start;
+    int start, ret;
 
     p->reference = 3;
     p->buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
-    if (avctx->reget_buffer(avctx, p)) {
+    if ((ret = avctx->reget_buffer(avctx, p)) < 0) {
         av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     start = 48; /* hardcoded for now */
@@ -173,21 +173,34 @@ static int ir2_decode_frame(AVCodecContext *avctx,
     init_get_bits(&s->gb, buf + start, (buf_size - start) * 8);
 
     if (s->decode_delta) { /* intraframe */
-        ir2_decode_plane(s, avctx->width, avctx->height,
-                         s->picture.data[0], s->picture.linesize[0], ir2_luma_table);
+        if ((ret = ir2_decode_plane(s, avctx->width, avctx->height,
+                                    s->picture.data[0], s->picture.linesize[0],
+                                    ir2_luma_table)) < 0)
+            return ret;
+
         /* swapped U and V */
-        ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[2], s->picture.linesize[2], ir2_luma_table);
-        ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[1], s->picture.linesize[1], ir2_luma_table);
+        if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
+                                    s->picture.data[2], s->picture.linesize[2],
+                                    ir2_luma_table)) < 0)
+            return ret;
+        if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
+                                    s->picture.data[1], s->picture.linesize[1],
+                                    ir2_luma_table)) < 0)
+            return ret;
     } else { /* interframe */
-        ir2_decode_plane_inter(s, avctx->width, avctx->height,
-                         s->picture.data[0], s->picture.linesize[0], ir2_luma_table);
+        if ((ret = ir2_decode_plane_inter(s, avctx->width, avctx->height,
+                                          s->picture.data[0], s->picture.linesize[0],
+                                          ir2_luma_table)) < 0)
+            return ret;
         /* swapped U and V */
-        ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[2], s->picture.linesize[2], ir2_luma_table);
-        ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[1], s->picture.linesize[1], ir2_luma_table);
+        if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
+                                          s->picture.data[2], s->picture.linesize[2],
+                                          ir2_luma_table)) < 0)
+            return ret;
+        if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
+                                          s->picture.data[1], s->picture.linesize[1],
+                                          ir2_luma_table)) < 0)
+            return ret;
     }
 
     *picture   = s->picture;

libavcodec/rl2.c

@@ -71,9 +71,9 @@ static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
     uint8_t *line_end;
 
     /** copy start of the background frame */
-    for(i=0;i<=base_y;i++){
-        if(s->back_frame)
-            memcpy(out,back_frame,s->avctx->width);
+    for (i = 0; i <= base_y; i++) {
+        if (s->back_frame)
+            memcpy(out, back_frame, s->avctx->width);
         out        += stride;
         back_frame += s->avctx->width;
     }
@@ -82,40 +82,40 @@ static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
     out        += base_x - stride;
 
     /** decode the variable part of the frame */
-    while(in < in_end){
+    while (in < in_end) {
         uint8_t val = *in++;
         int len     = 1;
-        if(val >= 0x80){
-            if(in >= in_end)
+        if (val >= 0x80) {
+            if (in >= in_end)
                 break;
             len = *in++;
-            if(!len)
+            if (!len)
                 break;
         }
 
-        if(len >= out_end - out)
+        if (len >= out_end - out)
             break;
 
-        if(s->back_frame)
+        if (s->back_frame)
             val |= 0x80;
         else
             val &= ~0x80;
 
-        while(len--){
-            *out++ = (val == 0x80)? *back_frame:val;
+        while (len--) {
+            *out++ = (val == 0x80) ? *back_frame : val;
             back_frame++;
-            if(out == line_end){
+            if (out == line_end) {
                  out      += stride_adj;
                  line_end += stride;
-                 if(len >= out_end - out)
+                 if (len >= out_end - out)
                      break;
             }
         }
     }
 
     /** copy the rest from the background frame */
-    if(s->back_frame){
-        while(out < out_end){
+    if (s->back_frame) {
+        while (out < out_end) {
             memcpy(out, back_frame, line_end - out);
             back_frame += line_end - out;
             out         = line_end + stride_adj;
@@ -135,12 +135,13 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
     Rl2Context *s = avctx->priv_data;
     int back_size;
     int i;
+
     s->avctx       = avctx;
     avctx->pix_fmt = AV_PIX_FMT_PAL8;
     avcodec_get_frame_defaults(&s->frame);
 
     /** parse extra data */
-    if(!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE){
+    if (!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE) {
         av_log(avctx, AV_LOG_ERROR, "invalid extradata size\n");
         return AVERROR(EINVAL);
     }
@@ -149,24 +150,24 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
     s->video_base = AV_RL16(&avctx->extradata[0]);
     s->clr_count  = AV_RL32(&avctx->extradata[2]);
 
-    if(s->video_base >= avctx->width * avctx->height){
+    if (s->video_base >= avctx->width * avctx->height) {
         av_log(avctx, AV_LOG_ERROR, "invalid video_base\n");
         return AVERROR_INVALIDDATA;
     }
 
     /** initialize palette */
-    for(i=0;i<AVPALETTE_COUNT;i++)
+    for (i = 0; i < AVPALETTE_COUNT; i++)
         s->palette[i] = 0xFFU << 24 | AV_RB24(&avctx->extradata[6 + i * 3]);
 
     /** decode background frame if present */
     back_size = avctx->extradata_size - EXTRADATA1_SIZE;
 
-    if(back_size > 0){
+    if (back_size > 0) {
         uint8_t *back_frame = av_mallocz(avctx->width*avctx->height);
-        if(!back_frame)
+        if (!back_frame)
             return AVERROR(ENOMEM);
-        rl2_rle_decode(s,avctx->extradata + EXTRADATA1_SIZE,back_size,
-                           back_frame,avctx->width,0);
+        rl2_rle_decode(s, avctx->extradata + EXTRADATA1_SIZE, back_size,
+                       back_frame, avctx->width, 0);
         s->back_frame = back_frame;
     }
     return 0;
@@ -181,18 +182,19 @@ static int rl2_decode_frame(AVCodecContext *avctx,
     int ret, buf_size  = avpkt->size;
     Rl2Context *s = avctx->priv_data;
 
-    if(s->frame.data[0])
+    if (s->frame.data[0])
         avctx->release_buffer(avctx, &s->frame);
 
     /** get buffer */
-    s->frame.reference= 0;
+    s->frame.reference = 0;
     if ((ret = ff_get_buffer(avctx, &s->frame)) < 0) {
         av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed\n");
         return ret;
     }
 
     /** run length decode */
-    rl2_rle_decode(s,buf,buf_size,s->frame.data[0],s->frame.linesize[0],s->video_base);
+    rl2_rle_decode(s, buf, buf_size, s->frame.data[0], s->frame.linesize[0],
+                   s->video_base);
 
     /** make the palette available on the way out */
     memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE);
@@ -214,7 +216,7 @@ static av_cold int rl2_decode_end(AVCodecContext *avctx)
 {
     Rl2Context *s = avctx->priv_data;
 
-    if(s->frame.data[0])
+    if (s->frame.data[0])
         avctx->release_buffer(avctx, &s->frame);
 
     av_free(s->back_frame);