avcodec/mpeg4videodec: avoid invalid values and reinitialize in format changes for studio profile

Fixes: out of array access
Fixes: 23327/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG4_fuzzer-5134822992510976

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpegSigned-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit e53235f06c229a23d3241b47e32647019161fb7c)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/mpeg4videodec.c

@@ -3134,6 +3134,7 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     MpegEncContext *s = &ctx->m;
     int width, height;
     int bits_per_raw_sample;
+    int rgb, chroma_format;
 
             // random_accessible_vol and video_object_type_indication have already
             // been read by the caller decode_vol_header()
@@ -3141,28 +3142,36 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
             ctx->shape = get_bits(gb, 2); /* video_object_layer_shape */
             skip_bits(gb, 4); /* video_object_layer_shape_extension */
             skip_bits1(gb); /* progressive_sequence */
+            if (ctx->shape != RECT_SHAPE) {
+                avpriv_request_sample(s->avctx, "MPEG-4 Studio profile non rectangular shape");
+                return AVERROR_PATCHWELCOME;
+            }
             if (ctx->shape != BIN_ONLY_SHAPE) {
-                ctx->rgb = get_bits1(gb); /* rgb_components */
-                s->chroma_format = get_bits(gb, 2); /* chroma_format */
-                if (!s->chroma_format) {
+                rgb = get_bits1(gb); /* rgb_components */
+                chroma_format = get_bits(gb, 2); /* chroma_format */
+                if (!chroma_format || chroma_format == CHROMA_420 || (rgb && chroma_format == CHROMA_422)) {
                     av_log(s->avctx, AV_LOG_ERROR, "illegal chroma format\n");
                     return AVERROR_INVALIDDATA;
                 }
 
                 bits_per_raw_sample = get_bits(gb, 4); /* bit_depth */
                 if (bits_per_raw_sample == 10) {
-                    if (ctx->rgb) {
+                    if (rgb) {
                         s->avctx->pix_fmt = AV_PIX_FMT_GBRP10;
                     }
                     else {
-                        s->avctx->pix_fmt = s->chroma_format == CHROMA_422 ? AV_PIX_FMT_YUV422P10 : AV_PIX_FMT_YUV444P10;
+                        s->avctx->pix_fmt = chroma_format == CHROMA_422 ? AV_PIX_FMT_YUV422P10 : AV_PIX_FMT_YUV444P10;
                     }
                 }
                 else {
                     avpriv_request_sample(s->avctx, "MPEG-4 Studio profile bit-depth %u", bits_per_raw_sample);
                     return AVERROR_PATCHWELCOME;
                 }
+                if (rgb != ctx->rgb || s->chroma_format != chroma_format)
+                    s->context_reinit = 1;
                 s->avctx->bits_per_raw_sample = bits_per_raw_sample;
+                ctx->rgb = rgb;
+                s->chroma_format = chroma_format;
             }
             if (ctx->shape == RECT_SHAPE) {
                 check_marker(s->avctx, gb, "before video_object_layer_width");