Full support for sending H.264 in RTP

This implements support for the "MP4" syntax of H.264 bitstreams.
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/rtpenc_h264.c

@@ -29,6 +29,24 @@
 #include "avc.h"
 #include "rtpenc.h"
 
+static const uint8_t *avc_mp4_find_startcode(const uint8_t *start, const uint8_t *end, int nal_lenght_size)
+{
+    int res = 0;
+
+    if (end - start < nal_lenght_size) {
+        return NULL;
+    }
+    while (nal_lenght_size--) {
+        res = (res << 8) | *start++;
+    }
+
+    if (res + start > end) {
+        return NULL;
+    }
+
+    return res + start;
+}
+
 static void nal_send(AVFormatContext *s1, const uint8_t *buf, int size, int last)
 {
     RTPMuxContext *s = s1->priv_data;
@@ -66,12 +84,20 @@ void ff_rtp_send_h264(AVFormatContext *s1, const uint8_t *buf1, int size)
     RTPMuxContext *s = s1->priv_data;
 
     s->timestamp = s->cur_timestamp;
-    r = ff_avc_find_startcode(buf1, buf1 + size);
+    r = s->nal_length_size ? (avc_mp4_find_startcode(buf1, buf1 + size, s->nal_length_size) ? buf1 : buf1 + size) : ff_avc_find_startcode(buf1, buf1 + size);
     while (r < buf1 + size) {
         const uint8_t *r1;
 
-        while(!*(r++));
-        r1 = ff_avc_find_startcode(r, buf1 + size);
+        if (s->nal_length_size) {
+            r1 = avc_mp4_find_startcode(r, buf1 + size, s->nal_length_size);
+            if (!r1) {
+                r1 = buf1 + size;
+            }
+            r += s->nal_length_size;
+        } else {
+            while(!*(r++));
+            r1 = ff_avc_find_startcode(r, buf1 + size);
+        }
         nal_send(s1, r, r1 - r, (r1 == buf1 + size));
         r = r1;
     }