Backport more TLS backend and ssld fixes & improvements from 3.6

openssl:
 * Don't manually initialise libssl 1.1.0 -- it does this automatically
 * SSL_library_init() should be called first otherwise
 * Move SSL_CTX construction to rb_setup_ssl_server()
 * Test for all required files (certificate & key) before doing anything
 * Free the old CTX before constructing a new one (Fixes #186)
 * Properly abort rb_setup_ssl_server() on CTX construction failures
 * Support ECDHE on more than one curve on OpenSSL 1.0.2 and above
 * Clean up ifdef indentation
 * Fix DH parameters memory leak

mbedtls:
 * Fix certificate fingerprint generation
 * Fix library linking order
 * Fix incorrect printf()-esque argument count
 * Return digest length for fingerprints instead of 1, consistent
   with the other backends

sslproc / ssld:
 * Fingerprint methods have no assocated file descriptors
 * Send TLS information (cipher, fingerprint) before data
 * Use correct header length for fingerprint method

Authored-by: Aaron Jones <aaronmdjones@gmail.com>
Authored-by: William Pitcock <nenolod@dereferenced.org>
Authored-by: Simon Arlott <sa.me.uk>

libratbox/configure

@@ -14986,7 +14986,7 @@ if ${ac_cv_lib_mbedtls_mbedtls_ssl_init+:} false; then :
   $as_echo_n "(cached) " >&6
 else
   ac_check_lib_save_LIBS=$LIBS
-LIBS="-lmbedtls -lmbedcrypto -lmbedx509 $LIBS"
+LIBS="-lmbedtls -lmbedx509 -lmbedcrypto $LIBS"
 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 
@@ -15018,7 +15018,7 @@ fi
 $as_echo "$ac_cv_lib_mbedtls_mbedtls_ssl_init" >&6; }
 if test "x$ac_cv_lib_mbedtls_mbedtls_ssl_init" = xyes; then :
 
-		MBEDTLS_LIBS="$MBEDTLS_LIBS -lmbedtls -lmbedcrypto -lmbedx509"
+		MBEDTLS_LIBS="$MBEDTLS_LIBS -lmbedtls -lmbedx509 -lmbedcrypto"
 		cf_enable_mbedtls=yes
 
 else

libratbox/configure.ac

@@ -351,9 +351,9 @@ if test "$cf_enable_mbedtls" != no; then
 	save_LIBS="$LIBS"
 	LIBS="$LIBS $MBEDTLS_LIBS"
 	AC_CHECK_LIB(mbedtls, mbedtls_ssl_init, [
-		MBEDTLS_LIBS="$MBEDTLS_LIBS -lmbedtls -lmbedcrypto -lmbedx509"
+		MBEDTLS_LIBS="$MBEDTLS_LIBS -lmbedtls -lmbedx509 -lmbedcrypto"
 		cf_enable_mbedtls=yes
-	], [cf_enable_mbedtls=no], [-lmbedcrypto -lmbedx509])
+	], [cf_enable_mbedtls=no], [-lmbedx509 -lmbedcrypto])
 fi
 
 dnl GnuTLS support

libratbox/src/mbedtls.c

@@ -552,12 +552,15 @@ rb_get_ssl_certfp(rb_fde_t *F, uint8_t certfp[RB_SSL_CERTFP_LEN], int method)
 	case RB_SSL_CERTFP_METH_SHA1:
 		md_type = MBEDTLS_MD_SHA1;
 		hashlen = RB_SSL_CERTFP_LEN_SHA1;
+		break;
 	case RB_SSL_CERTFP_METH_SHA256:
 		md_type = MBEDTLS_MD_SHA256;
 		hashlen = RB_SSL_CERTFP_LEN_SHA256;
+		break;
 	case RB_SSL_CERTFP_METH_SHA512:
 		md_type = MBEDTLS_MD_SHA512;
 		hashlen = RB_SSL_CERTFP_LEN_SHA512;
+		break;
 	default:
 		return 0;
 	}
@@ -572,13 +575,13 @@ rb_get_ssl_certfp(rb_fde_t *F, uint8_t certfp[RB_SSL_CERTFP_LEN], int method)
 
 	if ((ret = mbedtls_md(md_info, peer_cert->raw.p, peer_cert->raw.len, hash)) != 0)
 	{
-		rb_lib_log("rb_get_ssl_certfp: unable to get certfp for F: %p, -0x%x", -ret);
+		rb_lib_log("rb_get_ssl_certfp: unable to get certfp for F: %p, -0x%x", F, -ret);
 		return 0;
 	}
 
 	memcpy(certfp, hash, hashlen);
 
-	return 1;
+	return (int) hashlen;
 }
 
 int

libratbox/src/openssl.c

@@ -49,8 +49,20 @@
 #  endif
 #endif
 
-static SSL_CTX *ssl_server_ctx;
+/*
-static SSL_CTX *ssl_client_ctx;
+ * Use SSL_CTX_set_ecdh_auto() in OpenSSL 1.0.2 only
+ * Use SSL_CTX_set1_curves_list() in OpenSSL 1.0.2 and above
+ * TODO: Merge this into the block above if LibreSSL implements them
+ */
+#if !defined(LIBRESSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x10002000L)
+#  define LRB_HAVE_TLS_SET_CURVES 1
+#  if (OPENSSL_VERSION_NUMBER < 0x10100000L)
+#    define LRB_HAVE_TLS_ECDH_AUTO 1
+#  endif
+#endif
+
+static SSL_CTX *ssl_server_ctx = NULL;
+static SSL_CTX *ssl_client_ctx = NULL;
 static int libratbox_index = -1;
 
 static unsigned long
@@ -313,106 +325,35 @@ get_ssl_error(unsigned long err)
 int
 rb_init_ssl(void)
 {
-	int ret = 1;
 	char libratbox_data[] = "libratbox data";
-	const char libratbox_ciphers[] = "kEECDH+HIGH:kEDH+HIGH:HIGH:!RC4:!aNULL";
+
-	SSL_load_error_strings();
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
+	/*
+	 * OpenSSL 1.1.0 and above automatically initialises itself with sane defaults
+	 */
 	SSL_library_init();
+	SSL_load_error_strings();
+#endif
+
 	libratbox_index = SSL_get_ex_new_index(0, libratbox_data, NULL, NULL, NULL);
 
-#ifndef LRB_HAVE_TLS_METHOD_API
+	return 1;
-	ssl_server_ctx = SSL_CTX_new(SSLv23_server_method());
-#else
-	ssl_server_ctx = SSL_CTX_new(TLS_server_method());
-#endif
-
-	if(ssl_server_ctx == NULL)
-	{
-		rb_lib_log("rb_init_openssl: Unable to initialize OpenSSL server context: %s",
-			   get_ssl_error(ERR_get_error()));
-		ret = 0;
 }
 
-	long server_options = SSL_CTX_get_options(ssl_server_ctx);
-
-#ifndef LRB_HAVE_TLS_METHOD_API
-	server_options |= SSL_OP_NO_SSLv2;
-	server_options |= SSL_OP_NO_SSLv3;
-#endif
-
-#ifdef SSL_OP_SINGLE_DH_USE
-	server_options |= SSL_OP_SINGLE_DH_USE;
-#endif
-
-#ifdef SSL_OP_SINGLE_ECDH_USE
-	server_options |= SSL_OP_SINGLE_ECDH_USE;
-#endif
-
-#ifdef SSL_OP_NO_TICKET
-	server_options |= SSL_OP_NO_TICKET;
-#endif
-
-	server_options |= SSL_OP_CIPHER_SERVER_PREFERENCE;
-
-	SSL_CTX_set_options(ssl_server_ctx, server_options);
-	SSL_CTX_set_verify(ssl_server_ctx, SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE, verify_accept_all_cb);
-	SSL_CTX_set_session_cache_mode(ssl_server_ctx, SSL_SESS_CACHE_OFF);
-	SSL_CTX_set_cipher_list(ssl_server_ctx, libratbox_ciphers);
-
-	/* Set ECDHE on OpenSSL 1.00+, but make sure it's actually available because redhat are dicks
-	   and bastardise their OpenSSL for stupid reasons... */
-	#if (OPENSSL_VERSION_NUMBER >= 0x10000000L) && defined(NID_secp384r1)
-		EC_KEY *key = EC_KEY_new_by_curve_name(NID_secp384r1);
-		if (key) {
-			SSL_CTX_set_tmp_ecdh(ssl_server_ctx, key);
-			EC_KEY_free(key);
-		}
-	#endif
-
-#ifndef LRB_HAVE_TLS_METHOD_API
-	ssl_client_ctx = SSL_CTX_new(SSLv23_client_method());
-#else
-	ssl_client_ctx = SSL_CTX_new(TLS_client_method());
-#endif
-
-	if(ssl_client_ctx == NULL)
-	{
-		rb_lib_log("rb_init_openssl: Unable to initialize OpenSSL client context: %s",
-			   get_ssl_error(ERR_get_error()));
-		ret = 0;
-	}
-
-#ifndef LRB_HAVE_TLS_METHOD_API
-	SSL_CTX_set_options(ssl_client_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
-#endif
-
-#ifdef SSL_OP_NO_TICKET
-	SSL_CTX_set_options(ssl_client_ctx, SSL_OP_NO_TICKET);
-#endif
-
-	SSL_CTX_set_cipher_list(ssl_client_ctx, libratbox_ciphers);
-
-	return ret;
-}
-
-
 int
 rb_setup_ssl_server(const char *cert, const char *keyfile, const char *dhfile, const char *cipher_list)
 {
-	DH *dh;
+	const char libratbox_ciphers[] = "kEECDH+HIGH:kEDH+HIGH:HIGH:!aNULL";
-	unsigned long err;
+
+	#ifdef LRB_HAVE_TLS_SET_CURVES
+	const char libratbox_curves[] = "P-521:P-384:P-256";
+	#endif
+
 	if(cert == NULL)
 	{
 		rb_lib_log("rb_setup_ssl_server: No certificate file");
 		return 0;
 	}
-	if(!SSL_CTX_use_certificate_chain_file(ssl_server_ctx, cert) || !SSL_CTX_use_certificate_chain_file(ssl_client_ctx, cert))
-	{
-		err = ERR_get_error();
-		rb_lib_log("rb_setup_ssl_server: Error loading certificate file [%s]: %s", cert,
-			   get_ssl_error(err));
-		return 0;
-	}
 
 	if(keyfile == NULL)
 	{
@@ -420,12 +361,96 @@ rb_setup_ssl_server(const char *cert, const char *keyfile, const char *dhfile, c
 		return 0;
 	}
 
+	if(cipher_list == NULL)
+		cipher_list = libratbox_ciphers;
+
+	if(ssl_server_ctx)
+		SSL_CTX_free(ssl_server_ctx);
+
+	if(ssl_client_ctx)
+		SSL_CTX_free(ssl_client_ctx);
+
+	#ifdef LRB_HAVE_TLS_METHOD_API
+	ssl_server_ctx = SSL_CTX_new(TLS_server_method());
+	ssl_client_ctx = SSL_CTX_new(TLS_client_method());
+	#else
+	ssl_server_ctx = SSL_CTX_new(SSLv23_server_method());
+	ssl_client_ctx = SSL_CTX_new(SSLv23_client_method());
+	#endif
+
+	if(ssl_server_ctx == NULL)
+	{
+		rb_lib_log("rb_init_openssl: Unable to initialize OpenSSL server context: %s",
+			   get_ssl_error(ERR_get_error()));
+		return 0;
+	}
+
+	if(ssl_client_ctx == NULL)
+	{
+		rb_lib_log("rb_init_openssl: Unable to initialize OpenSSL client context: %s",
+			   get_ssl_error(ERR_get_error()));
+		return 0;
+	}
+
+	#ifndef LRB_HAVE_TLS_METHOD_API
+	SSL_CTX_set_options(ssl_server_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
+	SSL_CTX_set_options(ssl_client_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3);
+	#endif
+
+	#ifdef SSL_OP_SINGLE_DH_USE
+	SSL_CTX_set_options(ssl_server_ctx, SSL_OP_SINGLE_DH_USE);
+	#endif
+
+	#ifdef SSL_OP_SINGLE_ECDH_USE
+	SSL_CTX_set_options(ssl_server_ctx, SSL_OP_SINGLE_ECDH_USE);
+	#endif
+
+	#ifdef SSL_OP_NO_TICKET
+	SSL_CTX_set_options(ssl_server_ctx, SSL_OP_NO_TICKET);
+	SSL_CTX_set_options(ssl_client_ctx, SSL_OP_NO_TICKET);
+	#endif
+
+	#ifdef SSL_OP_CIPHER_SERVER_PREFERENCE
+	SSL_CTX_set_options(ssl_server_ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
+	#endif
+
+	#ifdef LRB_HAVE_TLS_ECDH_AUTO
+	SSL_CTX_set_ecdh_auto(ssl_server_ctx, 1);
+	#endif
+
+	#ifdef LRB_HAVE_TLS_SET_CURVES
+	SSL_CTX_set1_curves_list(ssl_server_ctx, libratbox_curves);
+	SSL_CTX_set1_curves_list(ssl_client_ctx, libratbox_curves);
+	#endif
+
+	SSL_CTX_set_verify(ssl_server_ctx, SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE, verify_accept_all_cb);
+	SSL_CTX_set_session_cache_mode(ssl_server_ctx, SSL_SESS_CACHE_OFF);
+
+	/*
+	 * Set manual ECDHE curve on OpenSSL 1.0.0 & 1.0.1, but make sure it's actually available
+	 */
+	#if (OPENSSL_VERSION_NUMBER >= 0x10000000L) && (OPENSSL_VERSION_NUMBER < 0x10002000L) && !defined(OPENSSL_NO_ECDH)
+	EC_KEY *key = EC_KEY_new_by_curve_name(NID_secp384r1);
+	if(key) {
+		SSL_CTX_set_tmp_ecdh(ssl_server_ctx, key);
+		EC_KEY_free(key);
+	}
+	#endif
+
+	SSL_CTX_set_cipher_list(ssl_server_ctx, cipher_list);
+	SSL_CTX_set_cipher_list(ssl_client_ctx, cipher_list);
+
+	if(!SSL_CTX_use_certificate_chain_file(ssl_server_ctx, cert) || !SSL_CTX_use_certificate_chain_file(ssl_client_ctx, cert))
+	{
+		rb_lib_log("rb_setup_ssl_server: Error loading certificate file [%s]: %s", cert,
+			   get_ssl_error(ERR_get_error()));
+		return 0;
+	}
 
 	if(!SSL_CTX_use_PrivateKey_file(ssl_server_ctx, keyfile, SSL_FILETYPE_PEM) || !SSL_CTX_use_PrivateKey_file(ssl_client_ctx, keyfile, SSL_FILETYPE_PEM))
 	{
-		err = ERR_get_error();
 		rb_lib_log("rb_setup_ssl_server: Error loading keyfile [%s]: %s", keyfile,
-			   get_ssl_error(err));
+			   get_ssl_error(ERR_get_error()));
 		return 0;
 	}
 
@@ -435,32 +460,26 @@ rb_setup_ssl_server(const char *cert, const char *keyfile, const char *dhfile, c
 		BIO *bio = BIO_new_file(dhfile, "r");
 		if(bio != NULL)
 		{
-			dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL);
+			DH *dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL);
 			if(dh == NULL)
 			{
-				err = ERR_get_error();
 				rb_lib_log
 					("rb_setup_ssl_server: Error loading DH params file [%s]: %s",
-					 dhfile, get_ssl_error(err));
+					 dhfile, get_ssl_error(ERR_get_error()));
 				BIO_free(bio);
 				return 0;
 			}
 			BIO_free(bio);
 			SSL_CTX_set_tmp_dh(ssl_server_ctx, dh);
+			DH_free(dh);
 		}
 		else
 		{
-			err = ERR_get_error();
 			rb_lib_log("rb_setup_ssl_server: Error loading DH params file [%s]: %s",
-				   dhfile, get_ssl_error(err));
+				   dhfile, get_ssl_error(ERR_get_error()));
 		}
 	}
 
-	if (cipher_list != NULL)
-	{
-		SSL_CTX_set_cipher_list(ssl_server_ctx, cipher_list);
-	}
-
 	return 1;
 }
 
@@ -689,26 +708,14 @@ rb_get_ssl_strerror(rb_fde_t *F)
 int
 rb_get_ssl_certfp(rb_fde_t *F, uint8_t certfp[RB_SSL_CERTFP_LEN], int method)
 {
+	const EVP_MD *evp;
+	unsigned int len;
 	X509 *cert;
 	int res;
 
 	if(F->ssl == NULL)
 		return 0;
 
-	cert = SSL_get_peer_certificate((SSL *) F->ssl);
-	if(cert != NULL)
-	{
-		res = SSL_get_verify_result((SSL *) F->ssl);
-		if(
-			res == X509_V_OK ||
-			res == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN ||
-			res == X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE ||
-			res == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT ||
-			res == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY)
-		{
-			const EVP_MD *evp;
-			unsigned int len;
-
 	switch(method)
 	{
 	case RB_SSL_CERTFP_METH_SHA1:
@@ -727,14 +734,29 @@ rb_get_ssl_certfp(rb_fde_t *F, uint8_t certfp[RB_SSL_CERTFP_LEN], int method)
 		return 0;
 	}
 
-			X509_digest(cert, evp, certfp, &len);
+	cert = SSL_get_peer_certificate((SSL *) F->ssl);
-			X509_free(cert);
+	if(cert == NULL)
-			return len;
+		return 0;
-		}
+
+	res = SSL_get_verify_result((SSL *) F->ssl);
+	switch(res)
+	{
+	case X509_V_OK:
+	case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
+	case X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE:
+	case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT:
+	case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY:
+	case X509_V_ERR_CERT_UNTRUSTED:
+		break;
+	default:
 		X509_free(cert);
+		return 0;
 	}
 
-	return 0;
+	X509_digest(cert, evp, certfp, &len);
+	X509_free(cert);
+
+	return (int) len;
 }
 
 int

src/sslproc.c

@@ -422,7 +422,7 @@ ssl_process_certfp(ssl_ctl_t * ctl, ssl_ctl_buf_t * ctl_buf)
 	char *certfp_string;
 	int i;
 
-	if(ctl_buf->buflen > 5 + RB_SSL_CERTFP_LEN)
+	if(ctl_buf->buflen > 9 + RB_SSL_CERTFP_LEN)
 		return;		/* bogus message..drop it.. XXX should warn here */
 
 	fd = buf_to_uint32(&ctl_buf->buf[1]);
@@ -759,7 +759,6 @@ start_zlib_session(void *data)
 	rb_fde_t *F[2];
 	rb_fde_t *xF1, *xF2;
 	char *buf;
-	char buf2[9];
 	void *recvq_start;
 
 	size_t hdr = (sizeof(uint8_t) * 2) + sizeof(uint32_t);

ssld/ssld.c

@@ -708,10 +708,10 @@ ssl_process_accept_cb(rb_fde_t *F, int status, struct sockaddr *addr, rb_socklen
 
 	if(status == RB_OK)
 	{
-		conn_mod_read_cb(conn->mod_fd, conn);
-		conn_plain_read_cb(conn->plain_fd, conn);
 		ssl_send_cipher(conn);
 		ssl_send_certfp(conn);
+		conn_mod_read_cb(conn->mod_fd, conn);
+		conn_plain_read_cb(conn->plain_fd, conn);
 		return;
 	}
 	/* ircd doesn't care about the reason for this */
@@ -726,10 +726,10 @@ ssl_process_connect_cb(rb_fde_t *F, int status, void *data)
 
 	if(status == RB_OK)
 	{
-		conn_mod_read_cb(conn->mod_fd, conn);
-		conn_plain_read_cb(conn->plain_fd, conn);
 		ssl_send_cipher(conn);
 		ssl_send_certfp(conn);
+		conn_mod_read_cb(conn->mod_fd, conn);
+		conn_plain_read_cb(conn->plain_fd, conn);
 	}
 	else if(status == RB_ERR_TIMEOUT)
 		close_conn(conn, WAIT_PLAIN, "SSL handshake timed out");
@@ -1032,7 +1032,7 @@ mod_process_cmd_recv(mod_ctl_t * ctl)
 			}
 		case 'F':
 			{
-				if (ctl_buf->nfds != 2 || ctl_buf->buflen != 5)
+				if (ctl_buf->buflen != 5)
 				{
 					cleanup_bad_message(ctl, ctl_buf);
 					break;