tls: fix mbedTLS 2.7.0 build + handle sha256 failures

(mbedtls 2.70 compiled with MBEDTLS_DEPRECATED_REMOVED)

Closes #2453

10 files changed, 29 insertions, 10 deletions

diff --git a/lib/vtls/cyassl.c b/lib/vtls/cyassl.c
index 1bd42d2c8..913c22d4a 100644
--- a/lib/vtls/cyassl.c
+++ b/lib/vtls/cyassl.c
@@ -966,7 +966,7 @@ static CURLcode Curl_cyassl_random(struct Curl_easy *data,
   return CURLE_OK;
 }
 
-static void Curl_cyassl_sha256sum(const unsigned char *tmp, /* input */
+static CURLcode Curl_cyassl_sha256sum(const unsigned char *tmp, /* input */
                                   size_t tmplen,
                                   unsigned char *sha256sum /* output */,
                                   size_t unused)
@@ -976,6 +976,7 @@ static void Curl_cyassl_sha256sum(const unsigned char *tmp, /* input */
   InitSha256(&SHA256pw);
   Sha256Update(&SHA256pw, tmp, (word32)tmplen);
   Sha256Final(&SHA256pw, sha256sum);
+  return CURLE_OK;
 }
 
 static void *Curl_cyassl_get_internals(struct ssl_connect_data *connssl,
diff --git a/lib/vtls/darwinssl.c b/lib/vtls/darwinssl.c
index f445302fa..0919f10dc 100644
--- a/lib/vtls/darwinssl.c
+++ b/lib/vtls/darwinssl.c
@@ -2894,13 +2894,14 @@ static CURLcode Curl_darwinssl_md5sum(unsigned char *tmp, /* input */
   return CURLE_OK;
 }
 
-static void Curl_darwinssl_sha256sum(const unsigned char *tmp, /* input */
+static CURLcode Curl_darwinssl_sha256sum(const unsigned char *tmp, /* input */
                                      size_t tmplen,
                                      unsigned char *sha256sum, /* output */
                                      size_t sha256len)
 {
   assert(sha256len >= CURL_SHA256_DIGEST_LENGTH);
   (void)CC_SHA256(tmp, (CC_LONG)tmplen, sha256sum);
+  return CURLE_OK;
 }
 
 static bool Curl_darwinssl_false_start(void)
diff --git a/lib/vtls/gtls.c b/lib/vtls/gtls.c
index 078874103..3f30b6c8c 100644
--- a/lib/vtls/gtls.c
+++ b/lib/vtls/gtls.c
@@ -1761,7 +1761,7 @@ static CURLcode Curl_gtls_md5sum(unsigned char *tmp, /* input */
   return CURLE_OK;
 }
 
-static void Curl_gtls_sha256sum(const unsigned char *tmp, /* input */
+static CURLcode Curl_gtls_sha256sum(const unsigned char *tmp, /* input */
                                 size_t tmplen,
                                 unsigned char *sha256sum, /* output */
                                 size_t sha256len)
@@ -1778,6 +1778,7 @@ static void Curl_gtls_sha256sum(const unsigned char *tmp, /* input */
   memcpy(sha256sum, gcry_md_read(SHA256pw, 0), sha256len);
   gcry_md_close(SHA256pw);
 #endif
+  return CURLE_OK;
 }
 
 static bool Curl_gtls_cert_status_request(void)
diff --git a/lib/vtls/mbedtls.c b/lib/vtls/mbedtls.c
index 28251a388..e76e19b09 100644
--- a/lib/vtls/mbedtls.c
+++ b/lib/vtls/mbedtls.c
@@ -1023,13 +1023,20 @@ static bool Curl_mbedtls_data_pending(const struct connectdata *conn,
   return mbedtls_ssl_get_bytes_avail(&BACKEND->ssl) != 0;
 }
 
-static void Curl_mbedtls_sha256sum(const unsigned char *input,
+static CURLcode Curl_mbedtls_sha256sum(const unsigned char *input,
                                     size_t inputlen,
                                     unsigned char *sha256sum,
                                     size_t sha256len UNUSED_PARAM)
 {
   (void)sha256len;
+#if MBEDTLS_VERSION_NUMBER < 0x02070000
   mbedtls_sha256(input, inputlen, sha256sum, 0);
+#else
+  /* returns 0 on success, otherwise failure */
+  if(mbedtls_sha256_ret(input, inputlen, sha256sum, 0) != 0)
+    return CURLE_BAD_FUNCTION_ARGUMENT;
+#endif
+  return CURLE_OK;
 }
 
 static void *Curl_mbedtls_get_internals(struct ssl_connect_data *connssl,
diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c
index 458f9d814..edbacc671 100644
--- a/lib/vtls/nss.c
+++ b/lib/vtls/nss.c
@@ -2314,7 +2314,7 @@ static CURLcode Curl_nss_md5sum(unsigned char *tmp, /* input */
   return CURLE_OK;
 }
 
-static void Curl_nss_sha256sum(const unsigned char *tmp, /* input */
+static CURLcode Curl_nss_sha256sum(const unsigned char *tmp, /* input */
                                size_t tmplen,
                                unsigned char *sha256sum, /* output */
                                size_t sha256len)
@@ -2325,6 +2325,8 @@ static void Curl_nss_sha256sum(const unsigned char *tmp, /* input */
   PK11_DigestOp(SHA256pw, tmp, curlx_uztoui(tmplen));
   PK11_DigestFinal(SHA256pw, sha256sum, &SHA256out, curlx_uztoui(sha256len));
   PK11_DestroyContext(SHA256pw, PR_TRUE);
+
+  return CURLE_OK;
 }
 
 static bool Curl_nss_cert_status_request(void)
diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index fc9ad47ad..205d303ed 100644
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -3603,7 +3603,7 @@ static CURLcode Curl_ossl_md5sum(unsigned char *tmp, /* input */
 }
 
 #if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) && !defined(OPENSSL_NO_SHA256)
-static void Curl_ossl_sha256sum(const unsigned char *tmp, /* input */
+static CURLcode Curl_ossl_sha256sum(const unsigned char *tmp, /* input */
                                 size_t tmplen,
                                 unsigned char *sha256sum /* output */,
                                 size_t unused)
@@ -3617,6 +3617,7 @@ static void Curl_ossl_sha256sum(const unsigned char *tmp, /* input */
   EVP_DigestUpdate(mdctx, tmp, tmplen);
   EVP_DigestFinal_ex(mdctx, sha256sum, &len);
   EVP_MD_CTX_destroy(mdctx);
+  return CURLE_OK;
 }
 #endif
 
diff --git a/lib/vtls/polarssl.c b/lib/vtls/polarssl.c
index df29fa945..d36cc70ee 100644
--- a/lib/vtls/polarssl.c
+++ b/lib/vtls/polarssl.c
@@ -882,13 +882,14 @@ static bool Curl_polarssl_data_pending(const struct connectdata *conn,
   return ssl_get_bytes_avail(&BACKEND->ssl) != 0;
 }
 
-static void Curl_polarssl_sha256sum(const unsigned char *input,
+static CURLcode Curl_polarssl_sha256sum(const unsigned char *input,
                                     size_t inputlen,
                                     unsigned char *sha256sum,
                                     size_t sha256len UNUSED_PARAM)
 {
   (void)sha256len;
   sha256(input, inputlen, sha256sum, 0);
+  return CURLE_OK;
 }
 
 static void *Curl_polarssl_get_internals(struct ssl_connect_data *connssl,
diff --git a/lib/vtls/schannel.c b/lib/vtls/schannel.c
index b8afe46f1..76392a1fd 100644
--- a/lib/vtls/schannel.c
+++ b/lib/vtls/schannel.c
@@ -1949,13 +1949,14 @@ static CURLcode Curl_schannel_md5sum(unsigned char *input,
     return CURLE_OK;
 }
 
-static void Curl_schannel_sha256sum(const unsigned char *input,
+static CURLcode Curl_schannel_sha256sum(const unsigned char *input,
                                     size_t inputlen,
                                     unsigned char *sha256sum,
                                     size_t sha256len)
 {
     Curl_schannel_checksum(input, inputlen, sha256sum, sha256len,
                            PROV_RSA_AES, CALG_SHA_256);
+    return CURLE_OK;
 }
 
 static void *Curl_schannel_get_internals(struct ssl_connect_data *connssl,
diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c
index def1d30cb..7ad18a3df 100644
--- a/lib/vtls/vtls.c
+++ b/lib/vtls/vtls.c
@@ -831,8 +831,12 @@ CURLcode Curl_pin_peer_pubkey(struct Curl_easy *data,
     sha256sumdigest = malloc(CURL_SHA256_DIGEST_LENGTH);
     if(!sha256sumdigest)
       return CURLE_OUT_OF_MEMORY;
-    Curl_ssl->sha256sum(pubkey, pubkeylen,
+    encode = Curl_ssl->sha256sum(pubkey, pubkeylen,
                         sha256sumdigest, CURL_SHA256_DIGEST_LENGTH);
+
+    if(encode != CURLE_OK)
+      return encode;
+
     encode = Curl_base64_encode(data, (char *)sha256sumdigest,
                                 CURL_SHA256_DIGEST_LENGTH, &encoded,
                                 &encodedlen);
diff --git a/lib/vtls/vtls.h b/lib/vtls/vtls.h
index c5f9d4a3f..4f76cc9e1 100644
--- a/lib/vtls/vtls.h
+++ b/lib/vtls/vtls.h
@@ -72,7 +72,7 @@ struct Curl_ssl {
 
   CURLcode (*md5sum)(unsigned char *input, size_t inputlen,
                      unsigned char *md5sum, size_t md5sumlen);
-  void (*sha256sum)(const unsigned char *input, size_t inputlen,
+  CURLcode (*sha256sum)(const unsigned char *input, size_t inputlen,
                     unsigned char *sha256sum, size_t sha256sumlen);
 };