oma: check geob tag boundary

Prevent read after buffer boundary on corrupted tag.

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org

libavformat/omadec.c

@@ -115,13 +115,18 @@ static int kset(AVFormatContext *s, const uint8_t *r_val, const uint8_t *n_val,
     return 0;
 }
 
-static int rprobe(AVFormatContext *s, uint8_t *enc_header, const uint8_t *r_val)
+#define OMA_RPROBE_M_VAL 48 + 1
+
+static int rprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
+                  const uint8_t *r_val)
 {
     OMAContext *oc = s->priv_data;
     unsigned int pos;
     struct AVDES av_des;
 
-    if (!enc_header || !r_val)
+    if (!enc_header || !r_val ||
+        size < OMA_ENC_HEADER_SIZE + oc->k_size + oc->e_size + oc->i_size ||
+        size < OMA_RPROBE_M_VAL)
         return -1;
 
     /* m_val */
@@ -142,36 +147,41 @@ static int rprobe(AVFormatContext *s, uint8_t *enc_header, const uint8_t *r_val)
     return memcmp(&enc_header[pos], oc->sm_val, 8) ? -1 : 0;
 }
 
-static int nprobe(AVFormatContext *s, uint8_t *enc_header, int size,
+static int nprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
                   const uint8_t *n_val)
 {
     OMAContext *oc = s->priv_data;
-    uint32_t pos, taglen, datalen;
+    uint64_t pos;
+    uint32_t taglen, datalen;
     struct AVDES av_des;
 
-    if (!enc_header || !n_val)
+    if (!enc_header || !n_val ||
+        size < OMA_ENC_HEADER_SIZE + oc->k_size + 4)
         return -1;
 
     pos = OMA_ENC_HEADER_SIZE + oc->k_size;
     if (!memcmp(&enc_header[pos], "EKB ", 4))
         pos += 32;
 
+    if (size < pos + 44)
+        return -1;
+
     if (AV_RB32(&enc_header[pos]) != oc->rid)
         av_log(s, AV_LOG_DEBUG, "Mismatching RID\n");
 
     taglen  = AV_RB32(&enc_header[pos + 32]);
     datalen = AV_RB32(&enc_header[pos + 36]) >> 4;
 
-    if (taglen + (((uint64_t)datalen) << 4) + 44 > size)
-        return -1;
-
     pos += 44 + taglen;
 
+    if (datalen << 4 > size - pos)
+        return -1;
+
     av_des_init(&av_des, n_val, 192, 1);
     while (datalen-- > 0) {
         av_des_crypt(&av_des, oc->r_val, &enc_header[pos], 2, NULL, 1);
         kset(s, oc->r_val, NULL, 16);
-        if (!rprobe(s, enc_header, oc->r_val))
+        if (!rprobe(s, enc_header, size, oc->r_val))
             return 0;
         pos += 16;
     }
@@ -237,7 +247,7 @@ static int decrypt_init(AVFormatContext *s, ID3v2ExtraMeta *em, uint8_t *header)
         kset(s, s->key, s->key, s->keylen);
     }
     if (!memcmp(oc->r_val, (const uint8_t[8]){0}, 8) ||
-        rprobe(s, gdata, oc->r_val) < 0 &&
+        rprobe(s, gdata, geob->datasize, oc->r_val) < 0 &&
         nprobe(s, gdata, geob->datasize, oc->n_val) < 0) {
         int i;
         for (i = 0; i < FF_ARRAY_ELEMS(leaf_table); i += 2) {
@@ -245,7 +255,7 @@ static int decrypt_init(AVFormatContext *s, ID3v2ExtraMeta *em, uint8_t *header)
             AV_WL64(buf,     leaf_table[i]);
             AV_WL64(&buf[8], leaf_table[i + 1]);
             kset(s, buf, buf, 16);
-            if (!rprobe(s, gdata, oc->r_val) ||
+            if (!rprobe(s, gdata, geob->datasize, oc->r_val) ||
                 !nprobe(s, gdata, geob->datasize, oc->n_val))
                 break;
         }