ntlm: use strict order for SSL backend #if branches

With the recently introduced MultiSSL support multiple SSL backends
can be compiled into cURL That means that now the order of the SSL

One option would be to use the same SSL backend as was configured
via `curl_global_sslset()`, however, NTLMv2 support would appear
to be available only with some SSL backends. For example, when
eb88d778e (ntlm: Use Windows Crypt API, 2014-12-02) introduced
support for NTLMv1 using Windows' Crypt API, it specifically did
*not* introduce NTLMv2 support using Crypt API at the same time.

So let's select one specific SSL backend for NTLM support when
compiled with multiple SSL backends, using a priority order such
that we support NTLMv2 even if only one compiled-in SSL backend can
be used for that.

Ref: https://github.com/curl/curl/pull/1848

4 files changed, 73 insertions, 36 deletions

diff --git a/lib/curl_ntlm_core.c b/lib/curl_ntlm_core.c
index fa26813be..5154949e6 100644
--- a/lib/curl_ntlm_core.c
+++ b/lib/curl_ntlm_core.c
@@ -31,6 +31,25 @@
  * https://www.innovation.ch/java/ntlm.html
  */
 
+/* Please keep the SSL backend-specific #if branches in this order:
+
+   1. USE_OPENSSL
+   2. USE_GNUTLS_NETTLE
+   3. USE_GNUTLS
+   4. USE_NSS
+   5. USE_MBEDTLS
+   6. USE_DARWINSSL
+   7. USE_OS400CRYPTO
+   8. USE_WIN32_CRYPTO
+
+   This ensures that:
+   - the same SSL branch gets activated throughout this source
+     file even if multiple backends are enabled at the same time.
+   - OpenSSL and NSS have higher priority than Windows Crypt, due
+     to issues with the latter supporting NTLM2Session responses
+     in NTLM type-3 messages.
+ */
+
 #if !defined(USE_WINDOWS_SSPI) || defined(USE_WIN32_CRYPTO)
 
 #ifdef USE_OPENSSL
@@ -76,14 +95,6 @@
 #  define MD5_DIGEST_LENGTH 16
 #  define MD4_DIGEST_LENGTH 16
 
-#elif defined(USE_MBEDTLS)
-
-#  include <mbedtls/des.h>
-#  include <mbedtls/md4.h>
-#  if !defined(MBEDTLS_MD4_C)
-#    include "curl_md4.h"
-#  endif
-
 #elif defined(USE_NSS)
 
 #  include <nss.h>
@@ -92,6 +103,14 @@
 #  include "curl_md4.h"
 #  define MD5_DIGEST_LENGTH MD5_LENGTH
 
+#elif defined(USE_MBEDTLS)
+
+#  include <mbedtls/des.h>
+#  include <mbedtls/md4.h>
+#  if !defined(MBEDTLS_MD4_C)
+#    include "curl_md4.h"
+#  endif
+
 #elif defined(USE_DARWINSSL)
 
 #  include <CommonCrypto/CommonCryptor.h>
@@ -196,26 +215,6 @@ static void setup_des_key(const unsigned char *key_56,
   gcry_cipher_setkey(*des, key, sizeof(key));
 }
 
-#elif defined(USE_MBEDTLS)
-
-static bool encrypt_des(const unsigned char *in, unsigned char *out,
-                        const unsigned char *key_56)
-{
-  mbedtls_des_context ctx;
-  char key[8];
-
-  /* Expand the 56-bit key to 64-bits */
-  extend_key_56_to_64(key_56, key);
-
-  /* Set the key parity to odd */
-  mbedtls_des_key_set_parity((unsigned char *) key);
-
-  /* Perform the encryption */
-  mbedtls_des_init(&ctx);
-  mbedtls_des_setkey_enc(&ctx, (unsigned char *) key);
-  return mbedtls_des_crypt_ecb(&ctx, in, out) == 0;
-}
-
 #elif defined(USE_NSS)
 
 /*
@@ -281,6 +280,26 @@ fail:
   return rv;
 }
 
+#elif defined(USE_MBEDTLS)
+
+static bool encrypt_des(const unsigned char *in, unsigned char *out,
+                        const unsigned char *key_56)
+{
+  mbedtls_des_context ctx;
+  char key[8];
+
+  /* Expand the 56-bit key to 64-bits */
+  extend_key_56_to_64(key_56, key);
+
+  /* Set the key parity to odd */
+  mbedtls_des_key_set_parity((unsigned char *) key);
+
+  /* Perform the encryption */
+  mbedtls_des_init(&ctx);
+  mbedtls_des_setkey_enc(&ctx, (unsigned char *) key);
+  return mbedtls_des_crypt_ecb(&ctx, in, out) == 0;
+}
+
 #elif defined(USE_DARWINSSL)
 
 static bool encrypt_des(const unsigned char *in, unsigned char *out,
@@ -428,7 +447,7 @@ void Curl_ntlm_core_lm_resp(const unsigned char *keys,
   setup_des_key(keys + 14, &des);
   gcry_cipher_encrypt(des, results + 16, 8, plaintext, 8);
   gcry_cipher_close(des);
-#elif defined(USE_MBEDTLS) || defined(USE_NSS) || defined(USE_DARWINSSL) \
+#elif defined(USE_NSS) || defined(USE_MBEDTLS) || defined(USE_DARWINSSL) \
   || defined(USE_OS400CRYPTO) || defined(USE_WIN32_CRYPTO)
   encrypt_des(plaintext, results, keys);
   encrypt_des(plaintext, results + 8, keys + 7);
@@ -492,7 +511,7 @@ CURLcode Curl_ntlm_core_mk_lm_hash(struct Curl_easy *data,
     setup_des_key(pw + 7, &des);
     gcry_cipher_encrypt(des, lmbuffer + 8, 8, magic, 8);
     gcry_cipher_close(des);
-#elif defined(USE_MBEDTLS) || defined(USE_NSS) || defined(USE_DARWINSSL) \
+#elif defined(USE_NSS) || defined(USE_MBEDTLS) || defined(USE_DARWINSSL) \
   || defined(USE_OS400CRYPTO) || defined(USE_WIN32_CRYPTO)
     encrypt_des(magic, lmbuffer, pw);
     encrypt_des(magic, lmbuffer + 8, pw + 7);
@@ -571,13 +590,18 @@ CURLcode Curl_ntlm_core_mk_nt_hash(struct Curl_easy *data,
     gcry_md_write(MD4pw, pw, 2 * len);
     memcpy(ntbuffer, gcry_md_read(MD4pw, 0), MD4_DIGEST_LENGTH);
     gcry_md_close(MD4pw);
-#elif defined(USE_NSS) || defined(USE_OS400CRYPTO) || \
-    (defined(USE_MBEDTLS) && !defined(MBEDTLS_MD4_C))
+#elif defined(USE_NSS)
     Curl_md4it(ntbuffer, pw, 2 * len);
 #elif defined(USE_MBEDTLS)
+#if defined(MBEDTLS_MD4_C)
     mbedtls_md4(pw, 2 * len, ntbuffer);
+#else
+    Curl_md4it(ntbuffer, pw, 2 * len);
+#endif
 #elif defined(USE_DARWINSSL)
     (void)CC_MD4(pw, (CC_LONG)(2 * len), ntbuffer);
+#elif defined(USE_OS400CRYPTO)
+    Curl_md4it(ntbuffer, pw, 2 * len);
 #elif defined(USE_WIN32_CRYPTO)
     HCRYPTPROV hprov;
     if(CryptAcquireContext(&hprov, NULL, NULL, PROV_RSA_FULL,
diff --git a/lib/http_ntlm.c b/lib/http_ntlm.c
index c110fa717..28e638e69 100644
--- a/lib/http_ntlm.c
+++ b/lib/http_ntlm.c
@@ -41,7 +41,9 @@
 #include "vauth/vauth.h"
 #include "url.h"
 
-#if defined(USE_NSS)
+/* SSL backend-specific #if branches in this file must be kept in the order
+   documented in curl_ntlm_core. */
+#if defined(NTLM_NEEDS_NSS_INIT)
 #include "vtls/nssg.h"
 #elif defined(USE_WINDOWS_SSPI)
 #include "curl_sspi.h"
@@ -129,7 +131,7 @@ CURLcode Curl_output_ntlm(struct connectdata *conn, bool proxy)
   DEBUGASSERT(conn);
   DEBUGASSERT(conn->data);
 
-#ifdef USE_NSS
+#if defined(NTLM_NEEDS_NSS_INIT)
   if(CURLE_OK != Curl_nss_force_init(conn->data))
     return CURLE_OUT_OF_MEMORY;
 #endif
diff --git a/lib/vauth/ntlm.c b/lib/vauth/ntlm.c
index d38208ed7..50d922208 100644
--- a/lib/vauth/ntlm.c
+++ b/lib/vauth/ntlm.c
@@ -44,7 +44,9 @@
 #include "rand.h"
 #include "vtls/vtls.h"
 
-#ifdef USE_NSS
+/* SSL backend-specific #if branches in this file must be kept in the order
+   documented in curl_ntlm_core. */
+#if defined(NTLM_NEEDS_NSS_INIT)
 #include "vtls/nssg.h" /* for Curl_nss_force_init() */
 #endif
 
@@ -272,7 +274,7 @@ CURLcode Curl_auth_decode_ntlm_type2_message(struct Curl_easy *data,
   unsigned char *type2 = NULL;
   size_t type2_len = 0;
 
-#if defined(USE_NSS)
+#if defined(NTLM_NEEDS_NSS_INIT)
   /* Make sure the crypto backend is initialized */
   result = Curl_nss_force_init(data);
   if(result)
diff --git a/lib/vtls/vtls.h b/lib/vtls/vtls.h
index f1a11ea58..b85b365e5 100644
--- a/lib/vtls/vtls.h
+++ b/lib/vtls/vtls.h
@@ -124,6 +124,15 @@ CURLcode Curl_none_md5sum(unsigned char *input, size_t inputlen,
 #define ALPN_HTTP_1_1_LENGTH 8
 #define ALPN_HTTP_1_1 "http/1.1"
 
+/* If NTLM is the first available SSL backend (see order in curl_ntlm_core)
+   then it must be initialized to be used by NTLM. */
+#if !defined(USE_OPENSSL) && \
+    !defined(USE_GNUTLS_NETTLE) && \
+    !defined(USE_GNUTLS) && \
+    defined(USE_NSS)
+#define NTLM_NEEDS_NSS_INIT
+#endif
+
 /* set of helper macros for the backends to access the correct fields. For the
    proxy or for the remote host - to properly support HTTPS proxy */