In avcodec_open(), set return code to an error value only when an error occurs

instead of unconditionally at the start of the function.

This fixes a bug where a successful call to ff_thread_init() masks errors that
occur after that point in the function.  It also makes future bugs like this
less likely since the error code is now set near to the point in the code
where the error is found.

libavcodec/utils.c

@@ -456,7 +456,7 @@ AVFrame *avcodec_alloc_frame(void){
 
 int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
 {
-    int ret= -1;
+    int ret = 0;
 
     /* If there is a user-supplied mutex locking routine, call it. */
     if (ff_lockmgr_cb) {
@@ -467,11 +467,14 @@ int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
     entangled_thread_counter++;
     if(entangled_thread_counter != 1){
         av_log(avctx, AV_LOG_ERROR, "insufficient thread locking around avcodec_open/close()\n");
+        ret = -1;
         goto end;
     }
 
-    if(avctx->codec || !codec)
+    if(avctx->codec || !codec) {
+        ret = AVERROR(EINVAL);
         goto end;
+    }
 
     if (codec->priv_data_size > 0) {
       if(!avctx->priv_data){
@@ -521,6 +524,7 @@ int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
     if (avctx->codec_id != codec->id || (avctx->codec_type != codec->type
                            && avctx->codec_type != AVMEDIA_TYPE_ATTACHMENT)) {
         av_log(avctx, AV_LOG_ERROR, "codec type or id mismatches\n");
+        ret = AVERROR(EINVAL);
         goto free_and_end;
     }
     avctx->frame_number = 0;
@@ -535,6 +539,7 @@ int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
     if (avctx->codec->max_lowres < avctx->lowres) {
         av_log(avctx, AV_LOG_ERROR, "The maximum value for lowres supported by the decoder is %d\n",
                avctx->codec->max_lowres);
+        ret = AVERROR(EINVAL);
         goto free_and_end;
     }
     if (avctx->codec->sample_fmts && avctx->codec->encode) {
@@ -544,6 +549,7 @@ int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
                 break;
         if (avctx->codec->sample_fmts[i] == AV_SAMPLE_FMT_NONE) {
             av_log(avctx, AV_LOG_ERROR, "Specified sample_fmt is not supported.\n");
+            ret = AVERROR(EINVAL);
             goto free_and_end;
         }
     }
@@ -554,7 +560,6 @@ int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
             goto free_and_end;
         }
     }
-    ret=0;
 end:
     entangled_thread_counter--;