OpenSSL 3.0 compatibility

Edited by @aaronmdjones:

- Correct some data types and casts

- Minor style fixups (e.g. we put * on the variable name not the type)

- librb/src/openssl.c:

  - Defer call of BIO_free(3ssl) to the end of the conditional block
    to avoid having calls to it in multiple paths

  - Check the return value of SSL_CTX_set0_tmp_dh_pkey(3ssl) because if
    it fails then we must use EVP_PKEY_free(3ssl) to avoid a memory leak

    This could fail if, for example, the user supplied DSA parameters
    in the DH parameters file instead.

- ircd/newconf.c:

  - Check whether OSSL_DECODER_CTX_new_for_pkey(3ssl) was able to parse
    the given CHALLANGE public key as a valid RSA public key, and then
    check whether OSSL_DECODER_from_bio(3ssl) actually loads it
    successfully

- ircd/s_newconf.c:

  - Use EVP_PKEY_free(3ssl) instead of OPENSSL_free(3ssl) on EVP_PKEY
    pointers; this will avoid inadvertent memory leaks if the EVP_PKEY
    structure contains any dynamically-allocated child members

- modules/m_challenge.c:

  - Unconditionally use EVP(3ssl) to generate the SHA-1 digest of the
    random challenge; this API has been around for a very long time and
    is available in all supported versions of OpenSSL

  - Add lots of error checking to all steps of the process

Tested against 1.1.1 and 3.0; both with missing and provided DH parameters
(which works as you'd expect; the server will not negotiate a DHE cipher
without them), and CHALLENGE, including missing keys or keys of the wrong
type (e.g. when you supply an EdDSA key instead of an RSA key).

This does break compatibility with OpenSSL 1.1.0 and below, which are now
all end-of-life and unsupported anyway.

Closes #357

include/client.h

@@ -215,7 +215,7 @@ struct LocalUser
 	 */
 	char *passwd;
 	char *auth_user;
-	char *challenge;
+	unsigned char *challenge;
 	char *fullcaps;
 	char *cipher_string;
 

include/s_newconf.h

@@ -121,8 +121,12 @@ struct oper_conf
 
 #ifdef HAVE_LIBCRYPTO
 	char *rsa_pubkey_file;
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+	EVP_PKEY *rsa_pubkey;
+#else
 	RSA *rsa_pubkey;
 #endif
+#endif
 };
 
 extern struct remote_conf *make_remote_conf(void);

ircd/newconf.c

@@ -4,8 +4,13 @@
 #include "stdinc.h"
 
 #ifdef HAVE_LIBCRYPTO
+#include <openssl/evp.h>
 #include <openssl/pem.h>
 #include <openssl/rsa.h>
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+#include <openssl/decoder.h>
+#include <openssl/core.h>
+#endif
 #endif
 
 #include "newconf.h"
@@ -631,8 +636,26 @@ conf_end_oper(struct TopConf *tc)
 				return 0;
 			}
 
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+			OSSL_DECODER_CTX *const ctx = OSSL_DECODER_CTX_new_for_pkey(
+				&yy_tmpoper->rsa_pubkey, "PEM", NULL, "RSA",
+				OSSL_KEYMGMT_SELECT_PUBLIC_KEY, NULL, NULL);
+
+			if(ctx != NULL)
+			{
+				if(OSSL_DECODER_CTX_get_num_decoders(ctx) < 1 ||
+				   OSSL_DECODER_from_bio(ctx, file) < 1)
+				{
+					EVP_PKEY_free(yy_tmpoper->rsa_pubkey);
+					yy_tmpoper->rsa_pubkey = NULL;
+				}
+
+				OSSL_DECODER_CTX_free(ctx);
+			}
+#else
 			yy_tmpoper->rsa_pubkey =
 				(RSA *) PEM_read_bio_RSA_PUBKEY(file, NULL, 0, NULL);
+#endif
 
 			(void)BIO_set_close(file, BIO_CLOSE);
 			BIO_free(file);

ircd/s_newconf.c

@@ -31,6 +31,12 @@
  */
 
 #include "stdinc.h"
+
+#ifdef HAVE_LIBCRYPTO
+#include <openssl/evp.h>
+#include <openssl/rsa.h>
+#endif
+
 #include "ircd_defs.h"
 #include "s_conf.h"
 #include "s_newconf.h"
@@ -235,7 +241,11 @@ free_oper_conf(struct oper_conf *oper_p)
 	rb_free(oper_p->rsa_pubkey_file);
 
 	if(oper_p->rsa_pubkey)
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+		EVP_PKEY_free(oper_p->rsa_pubkey);
+#else
 		RSA_free(oper_p->rsa_pubkey);
+#endif
 #endif
 
 	rb_free(oper_p);

librb/src/openssl.c

@@ -392,27 +392,48 @@ rb_setup_ssl_server(const char *const certfile, const char *keyfile,
 	}
 	else
 	{
-		FILE *const dhf = fopen(dhfile, "r");
+		BIO *const dhf = BIO_new_file(dhfile, "r");
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+		EVP_PKEY *dhp = NULL;
+#else
 		DH *dhp = NULL;
+#endif
 
 		if(dhf == NULL)
 		{
-			rb_lib_log("%s: fopen ('%s'): %s", __func__, dhfile, strerror(errno));
+			rb_lib_log("%s: BIO_new_file ('%s'): %s", __func__, dhfile,
-		}
-		else if(PEM_read_DHparams(dhf, &dhp, NULL, NULL) == NULL)
-		{
-			rb_lib_log("%s: PEM_read_DHparams ('%s'): %s", __func__, dhfile,
 			           rb_ssl_strerror(rb_ssl_last_err()));
-			fclose(dhf);
 		}
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+		else if(PEM_read_bio_Parameters(dhf, &dhp) == NULL)
+		{
+			rb_lib_log("%s: PEM_read_bio_Parameters ('%s'): %s", __func__, dhfile,
+			           rb_ssl_strerror(rb_ssl_last_err()));
+		}
+#else
+		else if(PEM_read_bio_DHparams(dhf, &dhp, NULL, NULL) == NULL)
+		{
+			rb_lib_log("%s: PEM_read_bio_DHparams ('%s'): %s", __func__, dhfile,
+			           rb_ssl_strerror(rb_ssl_last_err()));
+		}
+#endif
 		else
 		{
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+			if(SSL_CTX_set0_tmp_dh_pkey(ssl_ctx_new, dhp) != 1)
+			{
+				rb_lib_log("%s: SSL_CTX_set0_tmp_dh_pkey ('%s'): %s", __func__, dhfile,
+				           rb_ssl_strerror(rb_ssl_last_err()));
+				EVP_PKEY_free(dhp);
+			}
+#else
 			SSL_CTX_set_tmp_dh(ssl_ctx_new, dhp);
 			DH_free(dhp);
-			fclose(dhf);
+#endif
 		}
-	}
 
+		BIO_free(dhf);
+	}
 
 	int ret_old = SSL_CTX_set_cipher_list(ssl_ctx_new, cipherlist);
 

modules/m_challenge.c

@@ -25,13 +25,11 @@
 #include "stdinc.h"
 
 #ifdef HAVE_LIBCRYPTO
-#include <openssl/pem.h>
+#  include <openssl/err.h>
-#include <openssl/rand.h>
+#  include <openssl/evp.h>
-#include <openssl/rsa.h>
+#  include <openssl/pem.h>
-#include <openssl/md5.h>
+#  include <openssl/rsa.h>
-#include <openssl/bn.h>
+#  include <openssl/sha.h>
-#include <openssl/evp.h>
-#include <openssl/err.h>
 #endif
 
 #include "client.h"
@@ -84,7 +82,11 @@ mapi_clist_av1 challenge_clist[] = { &challenge_msgtab, NULL };
 
 DECLARE_MODULE_AV2(challenge, NULL, NULL, challenge_clist, NULL, NULL, NULL, NULL, challenge_desc);
 
-static bool generate_challenge(char **r_challenge, char **r_response, RSA * key);
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+static bool generate_challenge(char **r_challenge, unsigned char **r_response, EVP_PKEY *key);
+#else
+static bool generate_challenge(char **r_challenge, unsigned char **r_response, RSA *key);
+#endif
 
 static void
 cleanup_challenge(struct Client *target_p)
@@ -294,48 +296,94 @@ m_challenge(struct MsgBuf *msgbuf_p, struct Client *client_p, struct Client *sou
 }
 
 static bool
-generate_challenge(char **r_challenge, char **r_response, RSA * rsa)
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+generate_challenge(char **r_challenge, unsigned char **r_response, EVP_PKEY *key)
+#else
+generate_challenge(char **r_challenge, unsigned char **r_response, RSA *key)
+#endif
 {
-	SHA_CTX ctx;
+	unsigned char secret[CHALLENGE_SECRET_LENGTH];
-	unsigned char secret[CHALLENGE_SECRET_LENGTH], *tmp;
+	unsigned char *tmp = NULL;
-	unsigned long length;
 	unsigned long e = 0;
 	unsigned long cnt = 0;
-	int ret;
+	bool retval = false;
+	size_t length;
+	EVP_MD_CTX *mctx = NULL;
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+	EVP_PKEY_CTX *pctx = NULL;
+#endif
 
-	if(!rsa)
+	if(!rb_get_random(secret, sizeof secret))
 		return false;
-	if(rb_get_random(secret, CHALLENGE_SECRET_LENGTH))
-	{
-		SHA1_Init(&ctx);
-		SHA1_Update(&ctx, (uint8_t *)secret, CHALLENGE_SECRET_LENGTH);
-		*r_response = rb_malloc(SHA_DIGEST_LENGTH);
-		SHA1_Final((uint8_t *)*r_response, &ctx);
 
-		length = RSA_size(rsa);
+	if((*r_response = rb_malloc(SHA_DIGEST_LENGTH)) == NULL)
-		tmp = rb_malloc(length);
+		return false;
-		ret = RSA_public_encrypt(CHALLENGE_SECRET_LENGTH, secret, tmp, rsa, RSA_PKCS1_OAEP_PADDING);
 
-		if(ret >= 0)
+	if((mctx = EVP_MD_CTX_new()) == NULL)
-		{
+		goto fail;
-			*r_challenge = (char *)rb_base64_encode(tmp, ret);
-			rb_free(tmp);
-			return true;
-		}
 
-		rb_free(tmp);
+	if(EVP_DigestInit(mctx, EVP_sha1()) < 1)
-		rb_free(*r_response);
+		goto fail;
-		*r_response = NULL;
-	}
 
-	ERR_load_crypto_strings();
+	if(EVP_DigestUpdate(mctx, secret, sizeof secret) < 1)
+		goto fail;
+
+	if(EVP_DigestFinal(mctx, *r_response, NULL) < 1)
+		goto fail;
+
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+	if((length = (size_t) EVP_PKEY_get_size(key)) < 1)
+		goto fail;
+
+	if((tmp = rb_malloc(length)) == NULL)
+		goto fail;
+
+	if((pctx = EVP_PKEY_CTX_new(key, NULL)) == NULL)
+		goto fail;
+
+	if(EVP_PKEY_encrypt_init(pctx) < 1)
+		goto fail;
+
+	if(EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_OAEP_PADDING) < 1)
+		goto fail;
+
+	if(EVP_PKEY_encrypt(pctx, tmp, &length, secret, sizeof secret) < 1)
+		goto fail;
+#else
+	if((length = (size_t) RSA_size(key)) < 1)
+		goto fail;
+
+	if((tmp = rb_malloc(length)) == NULL)
+		goto fail;
+
+	if(RSA_public_encrypt(sizeof secret, secret, tmp, key, RSA_PKCS1_OAEP_PADDING) < 1)
+		goto fail;
+#endif
+
+	if((*r_challenge = (char *) rb_base64_encode(tmp, (int) length)) == NULL)
+		goto fail;
+
+	retval = true;
+	goto done;
+
+fail:
 	while ((cnt < 100) && (e = ERR_get_error()))
 	{
-		ilog(L_MAIN, "SSL error: %s", ERR_error_string(e, 0));
+		ilog(L_MAIN, "OpenSSL Error (CHALLENGE): %s", ERR_error_string(e, 0));
 		cnt++;
 	}
 
-	return false;
+	rb_free(*r_response);
+	*r_response = NULL;
+
+done:
+	EVP_MD_CTX_free(mctx);
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+	EVP_PKEY_CTX_free(pctx);
+#endif
+	rb_free(tmp);
+
+	return retval;
 }
 
 #endif /* HAVE_LIBCRYPTO */