libavformat: add mbedTLS based TLS

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

7 files changed, 422 insertions, 3 deletions

diff --git a/Changelog b/Changelog
index 35b6c06..27a4d4e 100644
--- a/Changelog
+++ b/Changelog
@@ -24,6 +24,7 @@ version <next>:
 - Haivision SRT protocol via libsrt
 - Dropped support for building for Windows XP. The minimum supported Windows
   version is Windows Vista.
+- support mbedTLS-based TLS
 
 
 version 12:
diff --git a/configure b/configure
index 20a839c..848fe48 100755
--- a/configure
+++ b/configure
@@ -229,6 +229,7 @@ External library support:
   --enable-libxcb-shm        X11 shm communication [auto]
   --enable-libxcb-xfixes     X11 mouse rendering [auto]
   --enable-libxvid           MPEG-4 ASP video encoding
+  --enable-mbedtls           crypto
   --enable-openssl           crypto
   --enable-zlib              compression [autodetect]
 
@@ -1343,6 +1344,7 @@ EXTERNAL_LIBRARY_VERSION3_LIST="
     libopencore_amrwb
     libvo_aacenc
     libvo_amrwbenc
+    mbedtls
 "
 
 EXTERNAL_LIBRARY_LIST="
@@ -2506,7 +2508,7 @@ xcbgrab_indev_suggest="libxcb_shm libxcb_xfixes"
 
 # protocols
 ffrtmpcrypt_protocol_conflict="librtmp_protocol"
-ffrtmpcrypt_protocol_deps_any="gmp openssl"
+ffrtmpcrypt_protocol_deps_any="gmp mbedtls openssl"
 ffrtmpcrypt_protocol_select="tcp_protocol"
 ffrtmphttp_protocol_conflict="librtmp_protocol"
 ffrtmphttp_protocol_select="http_protocol"
@@ -2539,7 +2541,7 @@ sctp_protocol_deps="struct_sctp_event_subscribe"
 sctp_protocol_select="network"
 srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
-tls_protocol_deps_any="gnutls openssl"
+tls_protocol_deps_any="gnutls mbedtls openssl"
 tls_protocol_select="tcp_protocol"
 udp_protocol_select="network"
 unix_protocol_deps="sys_un_h"
@@ -2959,6 +2961,12 @@ fi
 enabled_all gnutls openssl &&
     die "GnuTLS and OpenSSL must not be enabled at the same time."
 
+enabled_all gnutls mbedtls &&
+    die "GnuTLS and mbedTLS must not be enabled at the same time."
+
+enabled_all openssl mbedtls &&
+    die "OpenSSL and mbedTLS must not be enabled at the same time."
+
 # Disable all the library-specific components if the library itself
 # is disabled, see AVCODEC_LIST and following _LIST variables.
 
@@ -4710,6 +4718,7 @@ enabled libx265           && require_pkg_config libx265 x265 x265.h x265_api_get
                              require_cpp_condition libx265 x265.h "X265_BUILD >= 57"
 enabled libxavs           && require libxavs "stdint.h xavs.h" xavs_encoder_encode -lxavs
 enabled libxvid           && require libxvid xvid.h xvid_global -lxvidcore
+enabled mbedtls           && require mbedtls mbedtls/ssl.h mbedtls_ssl_init -lmbedtls -lmbedcrypto -lmbedx509
 enabled mmal              && { check_lib mmal interface/mmal/mmal.h mmal_port_connect -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host ||
                                { ! enabled cross_compile &&
                                  add_cflags -isystem/opt/vc/include/ -isystem/opt/vc/include/interface/vmcs_host/linux -isystem/opt/vc/include/interface/vcos/pthreads -fgnu89-inline &&
diff --git a/libavformat/Makefile b/libavformat/Makefile
index 4f1e9ba..ce21ecb 100644
--- a/libavformat/Makefile
+++ b/libavformat/Makefile
@@ -408,6 +408,7 @@ OBJS-$(CONFIG_SCTP_PROTOCOL)             += sctp.o
 OBJS-$(CONFIG_SRTP_PROTOCOL)             += srtpproto.o srtp.o
 OBJS-$(CONFIG_TCP_PROTOCOL)              += tcp.o
 TLS-OBJS-$(CONFIG_GNUTLS)                += tls_gnutls.o
+TLS-OBJS-$(CONFIG_MBEDTLS)               += tls_mbedtls.o
 TLS-OBJS-$(CONFIG_OPENSSL)               += tls_openssl.o
 OBJS-$(CONFIG_TLS_PROTOCOL)              += tls.o $(TLS-OBJS-yes)
 OBJS-$(CONFIG_UDP_PROTOCOL)              += udp.o
diff --git a/libavformat/rtmpdh.c b/libavformat/rtmpdh.c
index 0593eac..9f7afda 100644
--- a/libavformat/rtmpdh.c
+++ b/libavformat/rtmpdh.c
@@ -132,6 +132,56 @@ static int bn_modexp(FFBigNum bn, FFBigNum y, FFBigNum q, FFBigNum p)
     BN_CTX_free(ctx);
     return 0;
 }
+#elif CONFIG_MBEDTLS
+#define bn_new(bn)                      \
+    do {                                \
+        bn = av_malloc(sizeof(*bn));    \
+        if (bn)                         \
+            mbedtls_mpi_init(bn);       \
+    } while (0)
+#define bn_free(bn)                     \
+    do {                                \
+        mbedtls_mpi_free(bn);           \
+        av_free(bn);                    \
+    } while (0)
+#define bn_set_word(bn, w)          mbedtls_mpi_lset(bn, w)
+#define bn_cmp(a, b)                mbedtls_mpi_cmp_mpi(a, b)
+#define bn_copy(to, from)           mbedtls_mpi_copy(to, from)
+#define bn_sub_word(bn, w)          mbedtls_mpi_sub_int(bn, bn, w)
+#define bn_cmp_1(bn)                mbedtls_mpi_cmp_int(bn, 1)
+#define bn_num_bytes(bn)            (mbedtls_mpi_bitlen(bn) + 7) / 8
+#define bn_bn2bin(bn, buf, len)     mbedtls_mpi_write_binary(bn, buf, len)
+#define bn_bin2bn(bn, buf, len)                     \
+    do {                                            \
+        bn_new(bn);                                 \
+        if (bn)                                     \
+            mbedtls_mpi_read_binary(bn, buf, len);  \
+    } while (0)
+#define bn_hex2bn(bn, buf, ret)                     \
+    do {                                            \
+        bn_new(bn);                                 \
+        if (bn)                                     \
+            ret = (mbedtls_mpi_read_string(bn, 16, buf) == 0);  \
+        else                                        \
+            ret = 1;                                \
+    } while (0)
+#define bn_random(bn, num_bits)                     \
+    do {                                            \
+        mbedtls_entropy_context entropy_ctx;        \
+        mbedtls_ctr_drbg_context ctr_drbg_ctx;      \
+                                                    \
+        mbedtls_entropy_init(&entropy_ctx);         \
+        mbedtls_ctr_drbg_init(&ctr_drbg_ctx);       \
+        mbedtls_ctr_drbg_seed(&ctr_drbg_ctx,        \
+                              mbedtls_entropy_func, \
+                              &entropy_ctx,         \
+                              NULL, 0);             \
+        mbedtls_mpi_fill_random(bn, (num_bits + 7) / 8, mbedtls_ctr_drbg_random, &ctr_drbg_ctx); \
+        mbedtls_ctr_drbg_free(&ctr_drbg_ctx);       \
+        mbedtls_entropy_free(&entropy_ctx);         \
+    } while (0)
+#define bn_modexp(bn, y, q, p)      mbedtls_mpi_exp_mod(bn, y, q, p, 0)
+
 #endif
 
 #define MAX_BYTES 18000
diff --git a/libavformat/rtmpdh.h b/libavformat/rtmpdh.h
index b4d6121..bce583a 100644
--- a/libavformat/rtmpdh.h
+++ b/libavformat/rtmpdh.h
@@ -35,6 +35,12 @@ typedef mpz_ptr FFBigNum;
 #include <openssl/dh.h>
 
 typedef BIGNUM *FFBigNum;
+#elif CONFIG_MBEDTLS
+#include <mbedtls/bignum.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+
+typedef mbedtls_mpi *FFBigNum;
 #endif
 
 typedef struct FF_DH {
diff --git a/libavformat/tls_mbedtls.c b/libavformat/tls_mbedtls.c
new file mode 100644
index 0000000..5e01af8
--- /dev/null
+++ b/libavformat/tls_mbedtls.c
@@ -0,0 +1,352 @@
+/*
+ * TLS/SSL Protocol
+ * Copyright (c) 2018 Thomas Volkert
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <mbedtls/certs.h>
+#include <mbedtls/config.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+#include <mbedtls/net.h>
+#include <mbedtls/platform.h>
+#include <mbedtls/ssl.h>
+#include <mbedtls/x509_crt.h>
+
+#include "libavutil/parseutils.h"
+
+#include "avformat.h"
+#include "internal.h"
+#include "tls.h"
+#include "url.h"
+
+typedef struct TLSContext {
+    const AVClass *class;
+    TLSShared tls_shared;
+    mbedtls_ssl_context ssl_context;
+    mbedtls_ssl_config ssl_config;
+    mbedtls_entropy_context entropy_context;
+    mbedtls_ctr_drbg_context ctr_drbg_context;
+    mbedtls_x509_crt ca_cert;
+    mbedtls_x509_crt own_cert;
+    mbedtls_pk_context priv_key;
+    char *priv_key_pw;
+} TLSContext;
+
+#define OFFSET(x) offsetof(TLSContext, x)
+
+static int tls_close(URLContext *h)
+{
+    TLSContext *tls_ctx = h->priv_data;
+
+    mbedtls_ssl_close_notify(&tls_ctx->ssl_context);
+    mbedtls_pk_free(&tls_ctx->priv_key);
+    mbedtls_x509_crt_free(&tls_ctx->ca_cert);
+    mbedtls_x509_crt_free(&tls_ctx->own_cert);
+    mbedtls_ssl_free(&tls_ctx->ssl_context);
+    mbedtls_ssl_config_free(&tls_ctx->ssl_config);
+    mbedtls_ctr_drbg_free(&tls_ctx->ctr_drbg_context);
+    mbedtls_entropy_free(&tls_ctx->entropy_context);
+
+    return 0;
+}
+
+static int handle_transport_error(URLContext *h, const char* func_name, int react_on_eagain, int ret)
+{
+    switch (ret) {
+    case AVERROR(EAGAIN):
+        return react_on_eagain;
+    case AVERROR_EXIT:
+        return 0;
+    case AVERROR(EPIPE):
+    case AVERROR(ECONNRESET):
+        return MBEDTLS_ERR_NET_CONN_RESET;
+    default:
+        av_log(h, AV_LOG_ERROR, "%s returned 0x%x\n", func_name, ret);
+        errno = EIO;
+        return MBEDTLS_ERR_NET_SEND_FAILED;
+    }
+}
+
+static int mbedtls_send(void *ctx, const unsigned char *buf, size_t len)
+{
+    URLContext *h = (URLContext*) ctx;
+    int ret = ffurl_write(h, buf, len);
+    if (ret >= 0)
+        return ret;
+
+    if (h->max_packet_size && len > h->max_packet_size)
+        return MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL;
+
+    return handle_transport_error(h, "ffurl_write", MBEDTLS_ERR_SSL_WANT_WRITE, ret);
+}
+
+static int mbedtls_recv(void *ctx, unsigned char *buf, size_t len)
+{
+    URLContext *h = ctx;
+    int ret = ffurl_read(h, buf, len);
+    if (ret >= 0)
+        return ret;
+
+    if (h->max_packet_size && len > h->max_packet_size)
+        return MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL;
+
+    return handle_transport_error(h, "ffurl_read", MBEDTLS_ERR_SSL_WANT_READ, ret);
+}
+
+static void handle_pk_parse_error(URLContext *h, int ret)
+{
+    switch (ret) {
+    case MBEDTLS_ERR_PK_FILE_IO_ERROR:
+        av_log(h, AV_LOG_ERROR, "Read of key file failed. Is it actually there, are the access permissions correct?\n");
+        break;
+    case MBEDTLS_ERR_PK_PASSWORD_REQUIRED:
+        av_log(h, AV_LOG_ERROR, "A password for the private key is missing.\n");
+        break;
+    case MBEDTLS_ERR_PK_PASSWORD_MISMATCH:
+        av_log(h, AV_LOG_ERROR, "The given password for the private key is wrong.\n");
+        break;
+    default:
+        av_log(h, AV_LOG_ERROR, "mbedtls_pk_parse_key returned -0x%x\n", -ret);
+        break;
+    }
+}
+
+static void handle_handshake_error(URLContext *h, int ret)
+{
+    switch (ret) {
+    case MBEDTLS_ERR_SSL_NO_USABLE_CIPHERSUITE:
+        av_log(h, AV_LOG_ERROR, "None of the common ciphersuites is usable. Was the local certificate set correctly?\n");
+        break;
+    case MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE:
+        av_log(h, AV_LOG_ERROR, "A fatal alert message was received from the peer. Does the peer have a correct certificate?\n");
+        break;
+    case MBEDTLS_ERR_SSL_CA_CHAIN_REQUIRED:
+        av_log(h, AV_LOG_ERROR, "No CA chain is set, but required to operate. Was the CA set correctly?\n");
+        break;
+    case MBEDTLS_ERR_NET_CONN_RESET:
+        av_log(h, AV_LOG_ERROR, "TLS handshake was aborted by peer.\n");
+        break;
+    default:
+        av_log(h, AV_LOG_ERROR, "mbedtls_ssl_handshake returned -0x%x\n", -ret);
+        break;
+    }
+}
+
+static void parse_options(TLSContext *tls_ctxc, const char *uri)
+{
+    char buf[1024];
+    const char *p = strchr(uri, '?');
+    if (!p)
+        return;
+
+    if (!tls_ctxc->priv_key_pw && av_find_info_tag(buf, sizeof(buf), "key_password", p))
+        tls_ctxc->priv_key_pw = av_strdup(buf);
+}
+
+static int tls_open(URLContext *h, const char *uri, int flags, AVDictionary **options)
+{
+    TLSContext *tls_ctx = h->priv_data;
+    TLSShared *shr = &tls_ctx->tls_shared;
+    uint32_t verify_res_flags;
+    int ret;
+
+    // parse additional options
+    parse_options(tls_ctx, uri);
+
+    if ((ret = ff_tls_open_underlying(shr, h, uri, options)) < 0)
+        goto fail;
+
+    mbedtls_ssl_init(&tls_ctx->ssl_context);
+    mbedtls_ssl_config_init(&tls_ctx->ssl_config);
+    mbedtls_entropy_init(&tls_ctx->entropy_context);
+    mbedtls_ctr_drbg_init(&tls_ctx->ctr_drbg_context);
+    mbedtls_x509_crt_init(&tls_ctx->ca_cert);
+    mbedtls_pk_init(&tls_ctx->priv_key);
+
+    // load trusted CA
+    if (shr->ca_file) {
+        if ((ret = mbedtls_x509_crt_parse_file(&tls_ctx->ca_cert, shr->ca_file)) != 0) {
+            av_log(h, AV_LOG_ERROR, "mbedtls_x509_crt_parse_file for CA cert returned %d\n", ret);
+            goto fail;
+        }
+    }
+
+    // load own certificate
+    if (shr->cert_file) {
+        if ((ret = mbedtls_x509_crt_parse_file(&tls_ctx->own_cert, shr->cert_file)) != 0) {
+            av_log(h, AV_LOG_ERROR, "mbedtls_x509_crt_parse_file for own cert returned %d\n", ret);
+            goto fail;
+        }
+    }
+
+    // load key file
+    if (shr->key_file) {
+        if ((ret = mbedtls_pk_parse_keyfile(&tls_ctx->priv_key,
+                                            shr->key_file,
+                                            tls_ctx->priv_key_pw)) != 0) {
+            handle_pk_parse_error(h, ret);
+            goto fail;
+        }
+    }
+
+    // seed the random number generator
+    if ((ret = mbedtls_ctr_drbg_seed(&tls_ctx->ctr_drbg_context,
+                                     mbedtls_entropy_func,
+                                     &tls_ctx->entropy_context,
+                                     NULL, 0)) != 0) {
+        av_log(h, AV_LOG_ERROR, "mbedtls_ctr_drbg_seed returned %d\n", ret);
+        goto fail;
+    }
+
+    if ((ret = mbedtls_ssl_config_defaults(&tls_ctx->ssl_config,
+                                           shr->listen ? MBEDTLS_SSL_IS_SERVER : MBEDTLS_SSL_IS_CLIENT,
+                                           MBEDTLS_SSL_TRANSPORT_STREAM,
+                                           MBEDTLS_SSL_PRESET_DEFAULT)) != 0) {
+        av_log(h, AV_LOG_ERROR, "mbedtls_ssl_config_defaults returned %d\n", ret);
+        goto fail;
+    }
+
+    mbedtls_ssl_conf_authmode(&tls_ctx->ssl_config,
+                              shr->ca_file ? MBEDTLS_SSL_VERIFY_REQUIRED : MBEDTLS_SSL_VERIFY_NONE);
+    mbedtls_ssl_conf_rng(&tls_ctx->ssl_config, mbedtls_ctr_drbg_random, &tls_ctx->ctr_drbg_context);
+    mbedtls_ssl_conf_ca_chain(&tls_ctx->ssl_config, &tls_ctx->ca_cert, NULL);
+
+    // set own certificate and private key
+    if ((ret = mbedtls_ssl_conf_own_cert(&tls_ctx->ssl_config, &tls_ctx->own_cert, &tls_ctx->priv_key)) != 0) {
+        av_log(h, AV_LOG_ERROR, "mbedtls_ssl_conf_own_cert returned %d\n", ret);
+        goto fail;
+    }
+
+    if ((ret = mbedtls_ssl_setup(&tls_ctx->ssl_context, &tls_ctx->ssl_config)) != 0) {
+        av_log(h, AV_LOG_ERROR, "mbedtls_ssl_setup returned %d\n", ret);
+        goto fail;
+    }
+
+    if (!shr->listen && !shr->numerichost) {
+        if ((ret = mbedtls_ssl_set_hostname(&tls_ctx->ssl_context, shr->host)) != 0) {
+            av_log(h, AV_LOG_ERROR, "mbedtls_ssl_set_hostname returned %d\n", ret);
+            goto fail;
+        }
+    }
+
+    // set I/O functions to use libavformat-internal code for transport layer
+    mbedtls_ssl_set_bio(&tls_ctx->ssl_context, shr->tcp, mbedtls_send, mbedtls_recv, NULL);
+
+    // ssl handshake
+    while ((ret = mbedtls_ssl_handshake(&tls_ctx->ssl_context)) != 0) {
+        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
+            handle_handshake_error(h, ret);
+            goto fail;
+        }
+    }
+
+    if (shr->verify) {
+        // check the result of the certificate verification
+        if ((verify_res_flags = mbedtls_ssl_get_verify_result(&tls_ctx->ssl_context)) != 0) {
+            av_log(h, AV_LOG_ERROR, "mbedtls_ssl_get_verify_result reported problems "\
+                                    "with the certificate verification, returned flags: %u\n",
+                                    verify_res_flags);
+            if (verify_res_flags & MBEDTLS_X509_BADCERT_NOT_TRUSTED)
+                av_log(h, AV_LOG_ERROR, "The certificate is not correctly signed by the trusted CA.\n");
+            goto fail;
+        }
+    }
+
+    return 0;
+
+fail:
+    tls_close(h);
+    return AVERROR(EIO);
+}
+
+static int handle_tls_error(URLContext *h, const char *func_name, int ret)
+{
+    switch (ret) {
+    case MBEDTLS_ERR_SSL_WANT_READ:
+    case MBEDTLS_ERR_SSL_WANT_WRITE:
+        return AVERROR(EAGAIN);
+    case MBEDTLS_ERR_NET_SEND_FAILED:
+    case MBEDTLS_ERR_NET_RECV_FAILED:
+        return AVERROR(EIO);
+    case MBEDTLS_ERR_NET_CONN_RESET:
+    case MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY:
+        av_log(h, AV_LOG_WARNING, "%s reported connection reset by peer\n", func_name);
+        return AVERROR_EOF;
+    default:
+        av_log(h, AV_LOG_ERROR, "%s returned -0x%x\n", func_name, -ret);
+        return AVERROR(EIO);
+    }
+}
+
+static int tls_read(URLContext *h, uint8_t *buf, int size)
+{
+    TLSContext *tls_ctx = h->priv_data;
+    int ret;
+
+    if ((ret = mbedtls_ssl_read(&tls_ctx->ssl_context, buf, size)) > 0) {
+        // return read length
+        return ret;
+    }
+
+    return handle_tls_error(h, "mbedtls_ssl_read", ret);
+}
+
+static int tls_write(URLContext *h, const uint8_t *buf, int size)
+{
+    TLSContext *tls_ctx = h->priv_data;
+    int ret;
+
+    if ((ret = mbedtls_ssl_write(&tls_ctx->ssl_context, buf, size)) > 0) {
+        // return written length
+        return ret;
+    }
+
+    return handle_tls_error(h, "mbedtls_ssl_write", ret);
+}
+
+static int tls_get_file_handle(URLContext *h)
+{
+    TLSContext *c = h->priv_data;
+    return ffurl_get_file_handle(c->tls_shared.tcp);
+}
+
+static const AVOption options[] = {
+    TLS_COMMON_OPTIONS(TLSContext, tls_shared), \
+    { "key_password", "Password for the private key file", OFFSET(priv_key_pw),  AV_OPT_TYPE_STRING, .flags = TLS_OPTFL }, \
+    { NULL }
+};
+
+static const AVClass tls_class = {
+    .class_name = "tls",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+const URLProtocol ff_tls_protocol = {
+    .name                = "tls",
+    .url_open2           = tls_open,
+    .url_read            = tls_read,
+    .url_write           = tls_write,
+    .url_close           = tls_close,
+    .url_get_file_handle = tls_get_file_handle,
+    .priv_data_size      = sizeof(TLSContext),
+    .flags               = URL_PROTOCOL_FLAG_NETWORK,
+    .priv_data_class     = &tls_class,
+};
diff --git a/libavformat/version.h b/libavformat/version.h
index 28fb076..016f266 100644
--- a/libavformat/version.h
+++ b/libavformat/version.h
@@ -30,7 +30,7 @@
 #include "libavutil/version.h"
 
 #define LIBAVFORMAT_VERSION_MAJOR 58
-#define LIBAVFORMAT_VERSION_MINOR  1
+#define LIBAVFORMAT_VERSION_MINOR  2
 #define LIBAVFORMAT_VERSION_MICRO  0
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \