obs-outputs: Add support for and use mbedTLS for SSL

This diff adds mbedTLS support.  PolarSSL and mbedTLS have grown so
different between 2015-or-so when libRTMP was written and now that it's
no longer feasible to just use the USE_POLARSSL flag that is present in
libRTMP already.  I kept the old PolarSSL flag, but now there's
USE_MBEDTLS instead which uses the library more responsibly due to the
introduction of a bunch of _init and _free functions, etc.

Right now we're statically linking, meaning that the dependency on
mbedTLS on the client machine is not necessary and which alleviates
concerns of shipping OBS with this flag on by default.

cmake/Modules/FindMbedTLS.cmake

@@ -0,0 +1,44 @@
+# Once done these will be defined:
+#
+#  MBEDTLS_LIB
+#  MBEDTLS_INCLUDE_DIRS
+#  MBEDTLS_LIBRARIES
+
+find_path(MBEDTLS_INCLUDE_DIR
+	NAMES mbedtls/ssl.h
+	HINTS
+		${_MBEDTLS_INCLUDE_DIRS}
+	PATHS
+		/usr/include /usr/local/include /opt/local/include)
+
+find_library(MBEDTLS_LIB
+	NAMES libmbedtls.a
+	HINTS
+		${_MBEDTLS_LIBRARY_DIRS}
+	PATHS
+		/usr/lib /usr/local/lib /opt/local/lib)
+
+find_library(MBEDCRYPTO_LIB
+	NAMES libmbedcrypto.a
+	HINTS
+		${_MBEDTLS_LIBRARY_DIRS}
+	PATHS
+		/usr/lib /usr/local/lib /opt/local/lib)
+
+find_library(MBEDX509_LIB
+	NAMES libmbedx509.a
+	HINTS
+		${_MBEDTLS_LIBRARY_DIRS}
+	PATHS
+		/usr/lib /usr/local/lib /opt/local/lib)
+
+mark_as_advanced(MBEDTLS_INCLUDE_DIR MBEDTLS_ARCH_INCLUDE_DIR MBEDTLS_LIB MBEDCRYPTO_LIB MBEDX509_LIB)
+
+if(MBEDTLS_LIB AND MBEDCRYPTO_LIB AND MBEDX509_LIB)
+	set(MBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR} ${MBEDTLS_ARCH_INCLUDE_DIR})
+	set(MBEDTLS_LIBRARIES ${MBEDTLS_LIB} ${MBEDCRYPTO_LIB} ${MBEDX509_LIB})
+	set(MBEDTLS_FOUND 1)
+else()
+  message (WARNING "Some mbed libs not found. TLS: ${MBEDTLS_LIB}, CRYPTO: ${MBEDCRYPTO_LIB}, x509: ${MBEDX509_LIB}")
+	set(MBEDTLS_FOUND 0)
+endif()

plugins/obs-outputs/CMakeLists.txt

@@ -1,20 +1,21 @@
 project(obs-outputs)
 
-option(USE_SSL "Enable rtmps support with OpenSSL" OFF)
+option(WITH_RTMPS "Enable rtmps support with mbedTLS" OFF)
 
-if (USE_SSL)
-	find_package(SSL QUIET)
+if (WITH_RTMPS)
+	find_package(MbedTLS QUIET)
 	find_package(ZLIB QUIET)
 endif()
 
-if (SSL_FOUND AND ZLIB_FOUND)
-	add_definitions(-DCRYPTO -DUSE_OPENSSL)
-	include_directories(${SSL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
+if (MBEDTLS_FOUND AND ZLIB_FOUND)
+	message (STATUS "mbedTLS libraries found: ${MBEDTLS_LIBRARIES}")
+	add_definitions(-DCRYPTO -DUSE_MBEDTLS)
+	include_directories(${MBEDTLS_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
 else()
-	if (USE_SSL)
-		message(WARNING "SSL enabled by user, but OpenSSL was not found")
+	if (WITH_RTMPS)
+		message(WARNING "RTMPS enabled by user, but mbedTLS was not found")
 	endif()
-	unset(SSL_LIBRARIES)
+	unset(MBEDTLS_LIBRARIES)
 	unset(ZLIB_LIBRARIES)
 	add_definitions(-DNO_CRYPTO)
 endif()
@@ -133,7 +134,7 @@ set(obs-outputs_SOURCES
 	flv-output.c
 	flv-mux.c
 	net-if.c)
-	
+
 add_library(obs-outputs MODULE
 	${ftl_SOURCES}
 	${ftl_HEADERS}
@@ -143,7 +144,7 @@ add_library(obs-outputs MODULE
 	${obs-outputs_librtmp_HEADERS})
 target_link_libraries(obs-outputs
 	libobs
-	${SSL_LIBRARIES}
+	${MBEDTLS_LIBRARIES}
 	${ZLIB_LIBRARIES}
 	${ftl_IMPORTS}
 	${obs-outputs_PLATFORM_DEPS})

plugins/obs-outputs/librtmp/dh.h

@@ -21,7 +21,59 @@
  *  http://www.gnu.org/copyleft/lgpl.html
  */
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/dhm.h>
+#include <mbedtls/bignum.h>
+typedef  mbedtls_mpi* MP_t;
+#define MP_new(m)	m = malloc(sizeof(mbedtls_mpi)); mbedtls_mpi_init(m)
+#define MP_set_w(mpi, w)	mbedtls_mpi_lset(mpi, w)
+#define MP_cmp(u, v)	mbedtls_mpi_cmp_mpi(u, v)
+#define MP_set(u, v)	mbedtls_mpi_copy(u, v)
+#define MP_sub_w(mpi, w)	mbedtls_mpi_sub_int(mpi, mpi, w)
+#define MP_cmp_1(mpi)	mbedtls_mpi_cmp_int(mpi, 1)
+#define MP_modexp(r, y, q, p)	mbedtls_mpi_exp_mod(r, y, q, p, NULL)
+#define MP_free(mpi)	mbedtls_mpi_free(mpi); free(mpi)
+#define MP_gethex(u, hex, res)	MP_new(u); res = mbedtls_mpi_read_string(u, 16, hex) == 0
+#define MP_bytes(u)	mbedtls_mpi_size(u)
+#define MP_setbin(u,buf,len)	mbedtls_mpi_write_binary(u,buf,len)
+#define MP_getbin(u,buf,len)	MP_new(u); mbedtls_mpi_read_binary(u,buf,len)
+
+typedef struct MDH
+{
+    MP_t p;
+    MP_t g;
+    MP_t pub_key;
+    MP_t priv_key;
+    long length;
+    mbedtls_dhm_context ctx;
+} MDH;
+
+#define MDH_new()	calloc(1,sizeof(MDH))
+#define MDH_free(vp)	{MDH *_dh = vp; mbedtls_dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);}
+
+static int MDH_generate_key(MDH *dh)
+{
+    unsigned char out[2];
+    MP_set(&dh->ctx.P, dh->p);
+    MP_set(&dh->ctx.G, dh->g);
+    dh->ctx.len = 128;
+    mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &RTMP_TLS_ctx->ctr_drbg);
+    MP_new(dh->pub_key);
+    MP_new(dh->priv_key);
+    MP_set(dh->pub_key, &dh->ctx.GX);
+    MP_set(dh->priv_key, &dh->ctx.X);
+    return 1;
+}
+
+static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
+{
+    MP_set(&dh->ctx.GY, pub);
+    size_t olen;
+    mbedtls_dhm_calc_secret(&dh->ctx, secret, len, &olen, NULL, NULL);
+    return 0;
+}
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/dhm.h>
 typedef mpi * MP_t;
 #define MP_new(m)	m = malloc(sizeof(mpi)); mpi_init(m)

plugins/obs-outputs/librtmp/handshake.h

@@ -24,7 +24,28 @@
 
 /* This file is #included in rtmp.c, it is not meant to be compiled alone */
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/md.h>
+#include <mbedtls/arc4.h>
+#ifndef SHA256_DIGEST_LENGTH
+#define SHA256_DIGEST_LENGTH	32
+#endif
+typedef mbedtls_md_context_t *HMAC_CTX;
+#define HMAC_setup(ctx, key, len)	ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \
+  mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \
+  mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len)
+#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len)
+#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig)
+#define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL
+
+typedef mbedtls_arc4_context*	RC4_handle;
+#define RC4_alloc(h)	*h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h)
+#define RC4_setkey(h,l,k)	mbedtls_arc4_setup(h,k,l)
+#define RC4_encrypt(h,l,d)	mbedtls_arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d)
+#define RC4_encrypt2(h,l,s,d)	mbedtls_arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d)
+#define RC4_free(h)	mbedtls_arc4_free(h); free(h); h = NULL
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/sha2.h>
 #include <polarssl/arc4.h>
 #ifndef SHA256_DIGEST_LENGTH
@@ -148,6 +169,8 @@ typedef unsigned int (getoff)(uint8_t *buf, unsigned int len);
 static unsigned int
 GetDHOffset2(uint8_t *handshake, unsigned int len)
 {
+    (void) len;
+
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 768;
     unsigned int res;
@@ -177,6 +200,8 @@ GetDHOffset2(uint8_t *handshake, unsigned int len)
 static unsigned int
 GetDigestOffset2(uint8_t *handshake, unsigned int len)
 {
+    (void) len;
+
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 772;
     unsigned int res;
@@ -206,6 +231,8 @@ GetDigestOffset2(uint8_t *handshake, unsigned int len)
 static unsigned int
 GetDHOffset1(uint8_t *handshake, unsigned int len)
 {
+    (void) len;
+
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 1532;
     unsigned int res;
@@ -235,6 +262,8 @@ GetDHOffset1(uint8_t *handshake, unsigned int len)
 static unsigned int
 GetDigestOffset1(uint8_t *handshake, unsigned int len)
 {
+    (void) len;
+
     unsigned int offset = 0;
     uint8_t *ptr = handshake + 8;
     unsigned int res;
@@ -1128,6 +1157,7 @@ HandShake(RTMP * r, int FP9HandShake)
                      __FUNCTION__);
         }
     }
+    // TODO(mgoulet): Should this have a HMAC_finish here?
 
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     return TRUE;
@@ -1482,6 +1512,8 @@ SHandShake(RTMP * r)
         }
     }
 
+    // TODO(mgoulet): Should this have an Rc4_free?
+
     RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
     return TRUE;
 }

plugins/obs-outputs/librtmp/hashswf.c

@@ -30,7 +30,20 @@
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 #endif
 
-#ifdef USE_POLARSSL
+#if defined(USE_MBEDTLS)
+#include <mbedtls/md.h>
+#ifndef SHA256_DIGEST_LENGTH
+#define SHA256_DIGEST_LENGTH	32
+#endif
+typedef mbedtls_md_context_t *HMAC_CTX;
+#define HMAC_setup(ctx, key, len)	ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \
+  mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \
+  mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len)
+#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len)
+#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig)
+#define HMAC_close(ctx) free(ctx); mbedtls_md_free(ctx); ctx = NULL
+
+#elif defined(USE_POLARSSL)
 #include <polarssl/sha2.h>
 #ifndef SHA256_DIGEST_LENGTH
 #define SHA256_DIGEST_LENGTH	32
@@ -40,6 +53,7 @@
 #define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
 #define HMAC_close(ctx)
+
 #elif defined(USE_GNUTLS)
 #include <nettle/hmac.h>
 #ifndef SHA256_DIGEST_LENGTH
@@ -51,6 +65,7 @@
 #define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
 #define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
 #define HMAC_close(ctx)
+
 #else	/* USE_OPENSSL */
 #include <openssl/ssl.h>
 #include <openssl/sha.h>
@@ -161,8 +176,17 @@ HTTP_get(struct HTTP_ctx *http, const char *url, HTTP_read_callback *cb)
         goto leave;
 #else
         TLS_client(RTMP_TLS_ctx, sb.sb_ssl);
+
+#if defined(USE_MBEDTLS)
+        mbedtls_net_context *server_fd = &RTMP_TLS_ctx->net;
+        server_fd->fd = sb.sb_socket;
+        TLS_setfd(sb.sb_ssl, server_fd);
+#else
         TLS_setfd(sb.sb_ssl, sb.sb_socket);
-        if (TLS_connect(sb.sb_ssl) < 0)
+#endif
+
+        int connect_return = TLS_connect(sb.sb_ssl);
+        if (connect_return < 0)
         {
             RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__);
             ret = HTTPRES_LOST_CONNECTION;