rtmp: Support AMF_DATA_TYPE_MIXEDARRAY

And fix the AMF_DATA_TYPE_ARRAY parsing while at it.

A MIXEDARRAY type, as the ARRAY, store the number of elements in
an uint32 before the list. The ARRAY is strict and does not have
an OBJECT terminator, MIXEDARRAY behaves like an OBJECT type and
a different than stated number of element can be present.

libavformat/rtmppkt.c

@@ -370,28 +370,35 @@ void ff_rtmp_packet_destroy(RTMPPacket *pkt)
 int ff_amf_tag_size(const uint8_t *data, const uint8_t *data_end)
 {
     const uint8_t *base = data;
+    AMFDataType type;
+    unsigned nb   = -1;
+    int parse_key = 1;
 
     if (data >= data_end)
         return -1;
-    switch (*data++) {
+    switch ((type = *data++)) {
     case AMF_DATA_TYPE_NUMBER:      return 9;
     case AMF_DATA_TYPE_BOOL:        return 2;
     case AMF_DATA_TYPE_STRING:      return 3 + AV_RB16(data);
     case AMF_DATA_TYPE_LONG_STRING: return 5 + AV_RB32(data);
     case AMF_DATA_TYPE_NULL:        return 1;
     case AMF_DATA_TYPE_ARRAY:
-        data += 4;
+        parse_key = 0;
+    case AMF_DATA_TYPE_MIXEDARRAY:
+        nb = bytestream_get_be32(&data);
     case AMF_DATA_TYPE_OBJECT:
-        for (;;) {
-            int size = bytestream_get_be16(&data);
+        while (nb-- > 0 || type != AMF_DATA_TYPE_ARRAY) {
             int t;
-            if (!size) {
-                data++;
-                break;
+            if (parse_key) {
+                int size = bytestream_get_be16(&data);
+                if (!size) {
+                    data++;
+                    break;
+                }
+                if (size < 0 || size >= data_end - data)
+                    return -1;
+                data += size;
             }
-            if (size < 0 || size >= data_end - data)
-                return -1;
-            data += size;
             t = ff_amf_tag_size(data, data_end);
             if (t < 0 || t >= data_end - data)
                 return -1;
@@ -474,12 +481,14 @@ static const char* rtmp_packet_type(int type)
 static void amf_tag_contents(void *ctx, const uint8_t *data,
                              const uint8_t *data_end)
 {
-    unsigned int size;
+    unsigned int size, nb = -1;
     char buf[1024];
+    AMFDataType type;
+    int parse_key = 1;
 
     if (data >= data_end)
         return;
-    switch (*data++) {
+    switch ((type = *data++)) {
     case AMF_DATA_TYPE_NUMBER:
         av_log(ctx, AV_LOG_DEBUG, " number %g\n", av_int2double(AV_RB64(data)));
         return;
@@ -488,7 +497,7 @@ static void amf_tag_contents(void *ctx, const uint8_t *data,
         return;
     case AMF_DATA_TYPE_STRING:
     case AMF_DATA_TYPE_LONG_STRING:
-        if (data[-1] == AMF_DATA_TYPE_STRING) {
+        if (type == AMF_DATA_TYPE_STRING) {
             size = bytestream_get_be16(&data);
         } else {
             size = bytestream_get_be32(&data);
@@ -502,22 +511,28 @@ static void amf_tag_contents(void *ctx, const uint8_t *data,
         av_log(ctx, AV_LOG_DEBUG, " NULL\n");
         return;
     case AMF_DATA_TYPE_ARRAY:
-        data += 4;
+        parse_key = 0;
+    case AMF_DATA_TYPE_MIXEDARRAY:
+        nb = bytestream_get_be32(&data);
     case AMF_DATA_TYPE_OBJECT:
         av_log(ctx, AV_LOG_DEBUG, " {\n");
-        for (;;) {
+        while (nb-- > 0 || type != AMF_DATA_TYPE_ARRAY) {
             int t;
-            size = bytestream_get_be16(&data);
-            av_strlcpy(buf, data, FFMIN(sizeof(buf), size + 1));
-            if (!size) {
-                av_log(ctx, AV_LOG_DEBUG, " }\n");
-                data++;
-                break;
+            if (parse_key) {
+                size = bytestream_get_be16(&data);
+                size = FFMIN(size, sizeof(buf) - 1);
+                if (!size) {
+                    av_log(ctx, AV_LOG_DEBUG, " }\n");
+                    data++;
+                    break;
+                }
+                memcpy(buf, data, size);
+                buf[size] = 0;
+                if (size >= data_end - data)
+                    return;
+                data += size;
+                av_log(ctx, AV_LOG_DEBUG, "  %s: ", buf);
             }
-            if (size >= data_end - data)
-                return;
-            data += size;
-            av_log(ctx, AV_LOG_DEBUG, "  %s: ", buf);
             amf_tag_contents(ctx, data, data_end);
             t = ff_amf_tag_size(data, data_end);
             if (t < 0 || t >= data_end - data)