rtmpdh: Use GMP functions directly, instead of nettle wrappers

mpz_import and mpz_export were added in GMP 4.1, in 2002.

This simplifies the DH code by clarifying that it only uses pure
bignum functions, no other parts of nettle/hogweed.
Signed-off-by: Martin Storsjö <martin@martin.st>

configure

@@ -1590,6 +1590,7 @@ CONFIG_EXTRA="
     fdctdsp
     fmtconvert
     gcrypt
+    gmp
     golomb
     gplv3
     h263dsp
@@ -1614,7 +1615,6 @@ CONFIG_EXTRA="
     mpegaudiodsp
     mpegvideo
     mpegvideoenc
-    nettle
     pixblockdsp
     qpeldsp
     qsv
@@ -2184,7 +2184,7 @@ x11grab_xcb_indev_deps="libxcb"
 
 # protocols
 ffrtmpcrypt_protocol_deps="!librtmp_protocol"
-ffrtmpcrypt_protocol_deps_any="gcrypt nettle openssl"
+ffrtmpcrypt_protocol_deps_any="gcrypt gmp openssl"
 ffrtmpcrypt_protocol_select="tcp_protocol"
 ffrtmphttp_protocol_deps="!librtmp_protocol"
 ffrtmphttp_protocol_select="http_protocol"
@@ -4336,7 +4336,7 @@ enabled openssl           && { check_lib openssl/ssl.h SSL_library_init -lssl -l
                                die "ERROR: openssl not found"; }
 
 if enabled gnutls; then
-    { check_lib nettle/bignum.h nettle_mpz_get_str_256 -lnettle -lhogweed -lgmp && enable nettle; } ||
+    { check_lib2 gmp.h mpz_export -lgmp && enable gmp; } ||
     { check_lib gcrypt.h gcry_mpi_new -lgcrypt && enable gcrypt; }
 fi
 

libavformat/rtmpdh.c

@@ -46,8 +46,8 @@
     "F71C35FDAD44CFD2D74F9208BE258FF324943328F67329C0" \
     "FFFFFFFFFFFFFFFF"
 
-#if CONFIG_NETTLE || CONFIG_GCRYPT
-#if CONFIG_NETTLE
+#if CONFIG_GMP || CONFIG_GCRYPT
+#if CONFIG_GMP
 #define bn_new(bn)                      \
     do {                                \
         bn = av_malloc(sizeof(*bn));    \
@@ -65,12 +65,17 @@
 #define bn_sub_word(bn, w)          mpz_sub_ui(bn, bn, w)
 #define bn_cmp_1(bn)                mpz_cmp_ui(bn, 1)
 #define bn_num_bytes(bn)            (mpz_sizeinbase(bn, 2) + 7) / 8
-#define bn_bn2bin(bn, buf, len)     nettle_mpz_get_str_256(len, buf, bn)
+#define bn_bn2bin(bn, buf, len)                     \
+    do {                                            \
+        memset(buf, 0, len);                        \
+        if (bn_num_bytes(bn) <= len)                \
+            mpz_export(buf, NULL, 1, 1, 0, 0, bn);  \
+    } while (0)
 #define bn_bin2bn(bn, buf, len)                     \
     do {                                            \
         bn_new(bn);                                 \
         if (bn)                                     \
-            nettle_mpz_set_str_256_u(bn, len, buf); \
+            mpz_import(bn, len, 1, 1, 0, 0, buf);   \
     } while (0)
 #define bn_hex2bn(bn, buf, ret)                     \
     do {                                            \

libavformat/rtmpdh.h

@@ -25,10 +25,9 @@
 #include "avformat.h"
 #include "config.h"
 
-#if CONFIG_NETTLE || CONFIG_GCRYPT
-#if CONFIG_NETTLE
+#if CONFIG_GMP || CONFIG_GCRYPT
+#if CONFIG_GMP
 #include <gmp.h>
-#include <nettle/bignum.h>
 
 typedef mpz_ptr FFBigNum;
 #elif CONFIG_GCRYPT