libavformat: add mbedTLS based TLS

Signed-off-by: James Almer <jamrial@gmail.com>

Changelog

@@ -8,6 +8,7 @@ version <next>:
 - fftdnoiz filter
 - aderivative and aintegral audio filters
 - pal75bars and pal100bars video filter sources
+- support mbedTLS based TLS
 
 
 version 4.0:

configure

@@ -213,7 +213,7 @@ External library support:
   --enable-gmp             enable gmp, needed for rtmp(t)e support
                            if openssl or librtmp is not used [no]
   --enable-gnutls          enable gnutls, needed for https support
-                           if openssl or libtls is not used [no]
+                           if openssl, libtls or mbedtls is not used [no]
   --disable-iconv          disable iconv [autodetect]
   --enable-jni             enable JNI support [no]
   --enable-ladspa          enable LADSPA audio filtering [no]
@@ -262,7 +262,7 @@ External library support:
   --enable-libtesseract    enable Tesseract, needed for ocr filter [no]
   --enable-libtheora       enable Theora encoding via libtheora [no]
   --enable-libtls          enable LibreSSL (via libtls), needed for https support
-                           if openssl or gnutls is not used [no]
+                           if openssl, gnutls or mbedtls is not used [no]
   --enable-libtwolame      enable MP2 encoding via libtwolame [no]
   --enable-libv4l2         enable libv4l2/v4l-utils [no]
   --enable-libvidstab      enable video stabilization using vid.stab [no]
@@ -291,13 +291,15 @@ External library support:
   --disable-lzma           disable lzma [autodetect]
   --enable-decklink        enable Blackmagic DeckLink I/O support [no]
   --enable-libndi_newtek   enable Newteck NDI I/O support [no]
+  --enable-mbedtls         enable mbedTLS, needed for https support
+                           if openssl, gnutls or libtls is not used [no]
   --enable-mediacodec      enable Android MediaCodec support [no]
   --enable-libmysofa       enable libmysofa, needed for sofalizer filter [no]
   --enable-openal          enable OpenAL 1.1 capture support [no]
   --enable-opencl          enable OpenCL processing [no]
   --enable-opengl          enable OpenGL rendering [no]
   --enable-openssl         enable openssl, needed for https support
-                           if gnutls or libtls is not used [no]
+                           if gnutls, libtls or mbedtls is not used [no]
   --disable-sndio          disable sndio support [autodetect]
   --disable-schannel       disable SChannel SSP, needed for TLS support on
                            Windows if openssl and gnutls are not used [autodetect]
@@ -1656,6 +1658,7 @@ EXTERNAL_LIBRARY_VERSION3_LIST="
     libopencore_amrwb
     libvmaf
     libvo_amrwbenc
+    mbedtls
     rkmpp
 "
 
@@ -3235,7 +3238,7 @@ xv_outdev_deps="xlib"
 async_protocol_deps="threads"
 bluray_protocol_deps="libbluray"
 ffrtmpcrypt_protocol_conflict="librtmp_protocol"
-ffrtmpcrypt_protocol_deps_any="gcrypt gmp openssl"
+ffrtmpcrypt_protocol_deps_any="gcrypt gmp openssl mbedtls"
 ffrtmpcrypt_protocol_select="tcp_protocol"
 ffrtmphttp_protocol_conflict="librtmp_protocol"
 ffrtmphttp_protocol_select="http_protocol"
@@ -3255,7 +3258,7 @@ librtmpt_protocol_deps="librtmp"
 librtmpte_protocol_deps="librtmp"
 libsmbclient_protocol_deps="libsmbclient gplv3"
 libssh_protocol_deps="libssh"
-libtls_conflict="openssl gnutls"
+libtls_conflict="openssl gnutls mbedtls"
 mmsh_protocol_select="http_protocol"
 mmst_protocol_select="network"
 libsrt_protocol_deps="libsrt"
@@ -3275,13 +3278,13 @@ rtmpte_protocol_suggest="zlib"
 rtmpts_protocol_select="ffrtmphttp_protocol https_protocol"
 rtmpts_protocol_suggest="zlib"
 rtp_protocol_select="udp_protocol"
-schannel_conflict="openssl gnutls libtls"
+schannel_conflict="openssl gnutls libtls mbedtls"
 sctp_protocol_deps="struct_sctp_event_subscribe struct_msghdr_msg_flags"
 sctp_protocol_select="network"
-securetransport_conflict="openssl gnutls libtls"
+securetransport_conflict="openssl gnutls libtls mbedtls"
 srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
-tls_protocol_deps_any="gnutls openssl schannel securetransport libtls"
+tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls"
 tls_protocol_select="tcp_protocol"
 udp_protocol_select="network"
 udplite_protocol_select="network"
@@ -3916,6 +3919,12 @@ fi
 enabled_all gnutls openssl &&
     die "GnuTLS and OpenSSL must not be enabled at the same time."
 
+enabled_all gnutls mbedtls &&
+    die "GnuTLS and mbedTLS must not be enabled at the same time."
+
+enabled_all openssl mbedtls &&
+    die "OpenSSL and mbedTLS must not be enabled at the same time."
+
 # Disable all the library-specific components if the library itself
 # is disabled, see AVCODEC_LIST and following _LIST variables.
 
@@ -6102,6 +6111,10 @@ enabled libzvbi           && require_pkg_config libzvbi zvbi-0.2 libzvbi.h vbi_d
                              { test_cpp_condition libzvbi.h "VBI_VERSION_MAJOR > 0 || VBI_VERSION_MINOR > 2 || VBI_VERSION_MINOR == 2 && VBI_VERSION_MICRO >= 28" ||
                                enabled gpl || die "ERROR: libzvbi requires version 0.2.28 or --enable-gpl."; }
 enabled libxml2           && require_pkg_config libxml2 libxml-2.0 libxml2/libxml/xmlversion.h xmlCheckVersion
+enabled mbedtls           && { check_pkg_config mbedtls mbedtls mbedtls/x509_crt.h mbedtls_x509_crt_init ||
+                               check_pkg_config mbedtls mbedtls mbedtls/ssl.h mbedtls_ssl_init ||
+                               check_lib mbedtls mbedtls/ssl.h mbedtls_ssl_init -lmbedtls ||
+                               die "ERROR: mbedTLS not found"; }
 enabled mediacodec        && { enabled jni || die "ERROR: mediacodec requires --enable-jni"; }
 enabled mmal              && { check_lib mmal interface/mmal/mmal.h mmal_port_connect -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host ||
                                { ! enabled cross_compile &&

libavformat/Makefile

@@ -609,6 +609,7 @@ OBJS-$(CONFIG_TEE_PROTOCOL)              += teeproto.o tee_common.o
 OBJS-$(CONFIG_TCP_PROTOCOL)              += tcp.o
 TLS-OBJS-$(CONFIG_GNUTLS)                += tls_gnutls.o
 TLS-OBJS-$(CONFIG_LIBTLS)                += tls_libtls.o
+TLS-OBJS-$(CONFIG_MBEDTLS)               += tls_mbedtls.o
 TLS-OBJS-$(CONFIG_OPENSSL)               += tls_openssl.o
 TLS-OBJS-$(CONFIG_SECURETRANSPORT)       += tls_securetransport.o
 TLS-OBJS-$(CONFIG_SCHANNEL)              += tls_schannel.o

libavformat/rtmpdh.c

@@ -38,6 +38,11 @@
 
 #include "rtmpdh.h"
 
+#if CONFIG_MBEDTLS
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+#endif
+
 #define P1024                                          \
     "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
     "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
@@ -159,6 +164,56 @@ static int bn_modexp(FFBigNum bn, FFBigNum y, FFBigNum q, FFBigNum p)
     BN_CTX_free(ctx);
     return 0;
 }
+#elif CONFIG_MBEDTLS
+#define bn_new(bn)                      \
+    do {                                \
+        bn = av_malloc(sizeof(*bn));    \
+        if (bn)                         \
+            mbedtls_mpi_init(bn);       \
+    } while (0)
+#define bn_free(bn)                     \
+    do {                                \
+        mbedtls_mpi_free(bn);           \
+        av_free(bn);                    \
+    } while (0)
+#define bn_set_word(bn, w)          mbedtls_mpi_lset(bn, w)
+#define bn_cmp(a, b)                mbedtls_mpi_cmp_mpi(a, b)
+#define bn_copy(to, from)           mbedtls_mpi_copy(to, from)
+#define bn_sub_word(bn, w)          mbedtls_mpi_sub_int(bn, bn, w)
+#define bn_cmp_1(bn)                mbedtls_mpi_cmp_int(bn, 1)
+#define bn_num_bytes(bn)            (mbedtls_mpi_bitlen(bn) + 7) / 8
+#define bn_bn2bin(bn, buf, len)     mbedtls_mpi_write_binary(bn, buf, len)
+#define bn_bin2bn(bn, buf, len)                     \
+    do {                                            \
+        bn_new(bn);                                 \
+        if (bn)                                     \
+            mbedtls_mpi_read_binary(bn, buf, len);  \
+    } while (0)
+#define bn_hex2bn(bn, buf, ret)                     \
+    do {                                            \
+        bn_new(bn);                                 \
+        if (bn)                                     \
+            ret = (mbedtls_mpi_read_string(bn, 16, buf) == 0);  \
+        else                                        \
+            ret = 1;                                \
+    } while (0)
+#define bn_random(bn, num_bits)                     \
+    do {                                            \
+        mbedtls_entropy_context entropy_ctx;        \
+        mbedtls_ctr_drbg_context ctr_drbg_ctx;      \
+                                                    \
+        mbedtls_entropy_init(&entropy_ctx);         \
+        mbedtls_ctr_drbg_init(&ctr_drbg_ctx);       \
+        mbedtls_ctr_drbg_seed(&ctr_drbg_ctx,        \
+                              mbedtls_entropy_func, \
+                              &entropy_ctx,         \
+                              NULL, 0);             \
+        mbedtls_mpi_fill_random(bn, (num_bits + 7) / 8, mbedtls_ctr_drbg_random, &ctr_drbg_ctx); \
+        mbedtls_ctr_drbg_free(&ctr_drbg_ctx);       \
+        mbedtls_entropy_free(&entropy_ctx);         \
+    } while (0)
+#define bn_modexp(bn, y, q, p)      mbedtls_mpi_exp_mod(bn, y, q, p, 0)
+
 #endif
 
 #define MAX_BYTES 18000

libavformat/rtmpdh.h

@@ -40,6 +40,11 @@ typedef gcry_mpi_t FFBigNum;
 #include <openssl/dh.h>
 
 typedef BIGNUM *FFBigNum;
+#elif CONFIG_MBEDTLS
+#include <mbedtls/bignum.h>
+
+typedef mbedtls_mpi *FFBigNum;
+
 #endif
 
 typedef struct FF_DH {

libavformat/version.h

@@ -32,7 +32,7 @@
 // Major bumping may affect Ticket5467, 5421, 5451(compatibility with Chromium)
 // Also please add any ticket numbers that you believe might be affected here
 #define LIBAVFORMAT_VERSION_MAJOR  58
-#define LIBAVFORMAT_VERSION_MINOR  15
+#define LIBAVFORMAT_VERSION_MINOR  16
 #define LIBAVFORMAT_VERSION_MICRO 100
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \