LibreSSL 3.5 compatibility

LibreSSL is now closer to OpenSSL 1.1 than OpenSSL 1.0. According to
https://undeadly.org/cgi?action=article;sid=20220116121253, this is the
intention of OpenBSD developers.

According to #40630, many special cases are needed to compile Tor against
LibreSSL 3.5 when using Tor's OpenSSL 1.0 compatibility mode, whereas only a
small number of #defines are required when using OpenSSL 1.1 compatibility
mode. One additional workaround is required for LibreSSL 3.4 compatibility.

Compiles and passes unit tests with LibreSSL 3.4.3 and 3.5.1.

configure.ac

@@ -1022,7 +1022,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 AC_MSG_CHECKING([for OpenSSL < 1.0.1])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
-#if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x1000100fL
+#if OPENSSL_VERSION_NUMBER < 0x1000100fL
 #error "too old"
 #endif
    ]], [[]])],

src/lib/crypt_ops/compat_openssl.h

@@ -20,32 +20,36 @@
  * \brief compatibility definitions for working with different openssl forks
  **/
 
-#if !defined(LIBRESSL_VERSION_NUMBER) && \
-  OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,1)
+#if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,1)
 #error "We require OpenSSL >= 1.0.1"
 #endif
 
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && \
-   ! defined(LIBRESSL_VERSION_NUMBER)
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
 /* We define this macro if we're trying to build with the majorly refactored
  * API in OpenSSL 1.1 */
 #define OPENSSL_1_1_API
 #endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && ... */
 
-#ifndef OPENSSL_1_1_API
-#define OpenSSL_version(v) SSLeay_version(v)
-#define tor_OpenSSL_version_num() SSLeay()
+/* LibreSSL claims to be OpenSSL 2.0 but lacks these OpenSSL 1.1 APIs */
+#if !defined(OPENSSL_1_1_API) || defined(LIBRESSL_VERSION_NUMBER)
 #define RAND_OpenSSL() RAND_SSLeay()
 #define STATE_IS_SW_SERVER_HELLO(st)       \
   (((st) == SSL3_ST_SW_SRVR_HELLO_A) ||    \
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else /* defined(OPENSSL_1_1_API) */
-#define tor_OpenSSL_version_num() OpenSSL_version_num()
+#else
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
+#endif
+
+/* OpenSSL 1.1 and LibreSSL both have these APIs */
+#ifndef OPENSSL_1_1_API
+#define OpenSSL_version(v) SSLeay_version(v)
+#define tor_OpenSSL_version_num() SSLeay()
+#else /* defined(OPENSSL_1_1_API) */
+#define tor_OpenSSL_version_num() OpenSSL_version_num()
 #endif /* !defined(OPENSSL_1_1_API) */
 
 #endif /* defined(ENABLE_OPENSSL) */

src/lib/crypt_ops/crypto_openssl_mgt.h

@@ -54,8 +54,7 @@
 #define DISABLE_ENGINES
 #endif
 
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \
-  !defined(LIBRESSL_VERSION_NUMBER)
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5)
 /* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require
  * setting up various callbacks.
  *

src/lib/crypt_ops/crypto_rsa_openssl.c

@@ -572,7 +572,9 @@ static bool
 rsa_private_key_too_long(RSA *rsa, int max_bits)
 {
   const BIGNUM *n, *e, *p, *q, *d, *dmp1, *dmq1, *iqmp;
-#ifdef OPENSSL_1_1_API
+#if defined(OPENSSL_1_1_API) && \
+    (!defined(LIBRESSL_VERSION_NUMBER) || \
+     LIBRESSL_VERSION_NUMBER >= OPENSSL_V_SERIES(3,5,0))
 
 #if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1)
   n = RSA_get0_n(rsa);
@@ -591,7 +593,7 @@ rsa_private_key_too_long(RSA *rsa, int max_bits)
 
   if (RSA_bits(rsa) > max_bits)
     return true;
-#else /* !defined(OPENSSL_1_1_API) */
+#else /* !defined(OPENSSL_1_1_API) && ... */
   n = rsa->n;
   e = rsa->e;
   p = rsa->p;
@@ -600,7 +602,7 @@ rsa_private_key_too_long(RSA *rsa, int max_bits)
   dmp1 = rsa->dmp1;
   dmq1 = rsa->dmq1;
   iqmp = rsa->iqmp;
-#endif /* defined(OPENSSL_1_1_API) */
+#endif /* defined(OPENSSL_1_1_API) && ... */
 
   if (n && BN_num_bits(n) > max_bits)
     return true;