vauth: Use CURLE_AUTH_ERROR for auth function errors

- Add new error code CURLE_AUTH_ERROR.

Prior to this change auth function errors were signaled by
CURLE_OUT_OF_MEMORY and CURLE_RECV_ERROR, and neither one was
technically correct.

Ref: https://github.com/curl/curl/pull/3848

Co-authored-by: Dominik Hölzl

Closes https://github.com/curl/curl/pull/3864

11 files changed, 62 insertions, 18 deletions

diff --git a/docs/libcurl/libcurl-errors.3 b/docs/libcurl/libcurl-errors.3
index 26def4fec..2697efd5b 100644
--- a/docs/libcurl/libcurl-errors.3
+++ b/docs/libcurl/libcurl-errors.3
@@ -254,6 +254,8 @@ Status returned failure when asked with \fICURLOPT_SSL_VERIFYSTATUS(3)\fP.
 Stream error in the HTTP/2 framing layer.
 .IP "CURLE_RECURSIVE_API_CALL (93)"
 An API function was called from inside a callback.
+.IP "CURLE_AUTH_ERROR (94)"
+An authentication function returned an error.
 .IP "CURLE_OBSOLETE*"
 These error codes will never be returned. They were used in an old libcurl
 version and are currently unused.
diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index cd264b785..9daad949f 100644
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -39,6 +39,7 @@ CURLCLOSEPOLICY_SLOWEST         7.7
 CURLE_ABORTED_BY_CALLBACK       7.1
 CURLE_AGAIN                     7.18.2
 CURLE_ALREADY_COMPLETE          7.7.2
+CURLE_AUTH_ERROR                7.66.0
 CURLE_BAD_CALLING_ORDER         7.1           7.17.0
 CURLE_BAD_CONTENT_ENCODING      7.10
 CURLE_BAD_DOWNLOAD_RESUME       7.10
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 66c27b773..f94afd7a3 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -600,6 +600,8 @@ typedef enum {
                                     */
   CURLE_RECURSIVE_API_CALL,      /* 93 - an api function was called from
                                     inside a callback */
+  CURLE_AUTH_ERROR,              /* 94 - an authentication function returned an
+                                    error */
   CURL_LAST /* never use! */
 } CURLcode;
 
diff --git a/lib/strerror.c b/lib/strerror.c
index e273f3765..d0650d80c 100644
--- a/lib/strerror.c
+++ b/lib/strerror.c
@@ -311,6 +311,9 @@ curl_easy_strerror(CURLcode error)
   case CURLE_RECURSIVE_API_CALL:
     return "API function called from within callback";
 
+  case CURLE_AUTH_ERROR:
+    return "An authentication function returned an error";
+
     /* error codes not used by current libcurl */
   case CURLE_OBSOLETE20:
   case CURLE_OBSOLETE24:
diff --git a/lib/vauth/digest_sspi.c b/lib/vauth/digest_sspi.c
index fe8093e8b..850d6262b 100644
--- a/lib/vauth/digest_sspi.c
+++ b/lib/vauth/digest_sspi.c
@@ -220,7 +220,10 @@ CURLcode Curl_auth_create_digest_md5_message(struct Curl_easy *data,
     free(output_token);
     free(input_token);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Base64 encode the response */
@@ -607,7 +610,10 @@ CURLcode Curl_auth_create_digest_http_message(struct Curl_easy *data,
 
       Curl_safefree(digest->http_context);
 
-      return CURLE_OUT_OF_MEMORY;
+      if(status == SEC_E_INSUFFICIENT_MEMORY)
+        return CURLE_OUT_OF_MEMORY;
+
+      return CURLE_AUTH_ERROR;
     }
 
     output_token_len = resp_buf.cbBuffer;
diff --git a/lib/vauth/krb5_gssapi.c b/lib/vauth/krb5_gssapi.c
index ea0a5f189..95bab0e2e 100644
--- a/lib/vauth/krb5_gssapi.c
+++ b/lib/vauth/krb5_gssapi.c
@@ -121,7 +121,7 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
 
       free(spn);
 
-      return CURLE_OUT_OF_MEMORY;
+      return CURLE_AUTH_ERROR;
     }
 
     free(spn);
@@ -168,7 +168,7 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
     Curl_gss_log_error(data, "gss_init_sec_context() failed: ",
                        major_status, minor_status);
 
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
   }
 
   if(output_token.value && output_token.length) {
@@ -252,7 +252,7 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
 
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Convert the username from internal format to a displayable token */
@@ -264,7 +264,7 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
 
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Setup the challenge "input" security buffer */
@@ -355,7 +355,7 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
 
     free(message);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Base64 encode the response */
diff --git a/lib/vauth/krb5_sspi.c b/lib/vauth/krb5_sspi.c
index 1f6e462bf..6ac049eb3 100644
--- a/lib/vauth/krb5_sspi.c
+++ b/lib/vauth/krb5_sspi.c
@@ -217,8 +217,12 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
   /* Free the decoded challenge as it is not required anymore */
   free(chlg);
 
+  if(status == SEC_E_INSUFFICIENT_MEMORY) {
+    return CURLE_OUT_OF_MEMORY;
+  }
+
   if(status != SEC_E_OK && status != SEC_I_CONTINUE_NEEDED) {
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
   }
 
   if(memcmp(&context, krb5->context, sizeof(context))) {
@@ -309,7 +313,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
   if(status != SEC_E_OK) {
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Get the fully qualified username back from the context */
@@ -319,7 +326,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
   if(status != SEC_E_OK) {
     free(chlg);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Setup the "input" security buffer */
@@ -438,7 +448,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
     free(message);
     free(trailer);
 
-    return CURLE_OUT_OF_MEMORY;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Allocate the encryption (wrap) buffer */
diff --git a/lib/vauth/ntlm_sspi.c b/lib/vauth/ntlm_sspi.c
index 589cca16c..28109f76a 100644
--- a/lib/vauth/ntlm_sspi.c
+++ b/lib/vauth/ntlm_sspi.c
@@ -169,8 +169,10 @@ CURLcode Curl_auth_create_ntlm_type1_message(struct Curl_easy *data,
   if(status == SEC_I_COMPLETE_NEEDED ||
     status == SEC_I_COMPLETE_AND_CONTINUE)
     s_pSecFn->CompleteAuthToken(ntlm->context, &type_1_desc);
+  else if(status == SEC_E_INSUFFICIENT_MEMORY)
+    return CURLE_OUT_OF_MEMORY;
   else if(status != SEC_E_OK && status != SEC_I_CONTINUE_NEEDED)
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
 
   /* Base64 encode the response */
   return Curl_base64_encode(data, (char *) ntlm->output_token,
@@ -316,7 +318,10 @@ CURLcode Curl_auth_create_ntlm_type3_message(struct Curl_easy *data,
     infof(data, "NTLM handshake failure (type-3 message): Status=%x\n",
           status);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Base64 encode the response */
diff --git a/lib/vauth/spnego_gssapi.c b/lib/vauth/spnego_gssapi.c
index 5d43e1100..f05afca96 100644
--- a/lib/vauth/spnego_gssapi.c
+++ b/lib/vauth/spnego_gssapi.c
@@ -121,7 +121,7 @@ CURLcode Curl_auth_decode_spnego_message(struct Curl_easy *data,
 
       free(spn);
 
-      return CURLE_OUT_OF_MEMORY;
+      return CURLE_AUTH_ERROR;
     }
 
     free(spn);
@@ -177,7 +177,7 @@ CURLcode Curl_auth_decode_spnego_message(struct Curl_easy *data,
     if(output_token.value)
       gss_release_buffer(&unused_status, &output_token);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Free previous token */
diff --git a/lib/vauth/spnego_sspi.c b/lib/vauth/spnego_sspi.c
index 4b21cc769..a4935276b 100644
--- a/lib/vauth/spnego_sspi.c
+++ b/lib/vauth/spnego_sspi.c
@@ -251,14 +251,25 @@ CURLcode Curl_auth_decode_spnego_message(struct Curl_easy *data,
     char buffer[STRERROR_LEN];
     failf(data, "InitializeSecurityContext failed: %s",
           Curl_sspi_strerror(nego->status, buffer, sizeof(buffer)));
-    return CURLE_OUT_OF_MEMORY;
+
+    if(nego->status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   if(nego->status == SEC_I_COMPLETE_NEEDED ||
      nego->status == SEC_I_COMPLETE_AND_CONTINUE) {
     nego->status = s_pSecFn->CompleteAuthToken(nego->context, &resp_desc);
     if(GSS_ERROR(nego->status)) {
-      return CURLE_RECV_ERROR;
+      char buffer[STRERROR_LEN];
+      failf(data, "CompleteAuthToken failed: %s",
+            Curl_sspi_strerror(nego->status, buffer, sizeof(buffer)));
+
+      if(nego->status == SEC_E_INSUFFICIENT_MEMORY)
+        return CURLE_OUT_OF_MEMORY;
+
+      return CURLE_AUTH_ERROR;
     }
   }
 
diff --git a/tests/data/test1538 b/tests/data/test1538
index 98d6731e9..9374debb7 100644
--- a/tests/data/test1538
+++ b/tests/data/test1538
@@ -126,7 +126,8 @@ e90: SSL public key does not match pinned public key
 e91: SSL server certificate status verification FAILED
 e92: Stream error in the HTTP/2 framing layer
 e93: API function called from within callback
-e94: Unknown error
+e94: An authentication function returned an error
+e95: Unknown error
 m-1: Please call curl_multi_perform() soon
 m0: No error
 m1: Invalid multi handle