rtmpdh: Pass the actual buffer size of the output secret key

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/rtmpcrypt.c

@@ -163,7 +163,7 @@ int ff_rtmpe_compute_secret_key(URLContext *h, const uint8_t *serverdata,
 
     /* compute the shared secret secret in order to compute RC4 keys */
     if ((ret = ff_dh_compute_shared_secret_key(rt->dh, serverdata + server_pos,
-                                               128, secret_key)) < 0)
+                                               128, secret_key, sizeof(secret_key))) < 0)
         return ret;
 
     /* set output key */

libavformat/rtmpdh.c

@@ -313,7 +313,8 @@ int ff_dh_write_public_key(FF_DH *dh, uint8_t *pub_key, int pub_key_len)
 }
 
 int ff_dh_compute_shared_secret_key(FF_DH *dh, const uint8_t *pub_key,
-                                    int pub_key_len, uint8_t *secret_key)
+                                    int pub_key_len, uint8_t *secret_key,
+                                    int secret_key_len)
 {
     FFBigNum q1 = NULL, pub_key_bn = NULL;
     int ret;
@@ -333,7 +334,7 @@ int ff_dh_compute_shared_secret_key(FF_DH *dh, const uint8_t *pub_key,
     /* when the public key is valid we have to compute the shared secret key */
     if ((ret = dh_is_valid_public_key(pub_key_bn, dh->p, q1)) < 0) {
         goto fail;
-    } else if ((ret = dh_compute_key(dh, pub_key_bn, pub_key_len,
+    } else if ((ret = dh_compute_key(dh, pub_key_bn, secret_key_len,
                                      secret_key)) < 0) {
         ret = AVERROR(EINVAL);
         goto fail;

libavformat/rtmpdh.h

@@ -92,11 +92,13 @@ int ff_dh_write_public_key(FF_DH *dh, uint8_t *pub_key, int pub_key_len);
  *
  * @param dh            a Diffie-Hellmann context, containing the private key
  * @param pub_key       the buffer containing the public key
- * @param pub_key_len   the length of the buffer
+ * @param pub_key_len   the length of the public key buffer
  * @param secret_key    the buffer where the secret key is written
+ * @param secret_key_len the length of the secret key buffer
  * @return length of the shared secret key on success, negative value otherwise
  */
 int ff_dh_compute_shared_secret_key(FF_DH *dh, const uint8_t *pub_key,
-                                    int pub_key_len, uint8_t *secret_key);
+                                    int pub_key_len, uint8_t *secret_key,
+                                    int secret_key_len);
 
 #endif /* AVFORMAT_RTMPDH_H */