vauth: Use CURLE_AUTH_ERROR for auth function errors

- Add new error code CURLE_AUTH_ERROR.

Prior to this change auth function errors were signaled by
CURLE_OUT_OF_MEMORY and CURLE_RECV_ERROR, and neither one was
technically correct.

Ref: https://github.com/curl/curl/pull/3848

Co-authored-by: Dominik Hölzl

Closes https://github.com/curl/curl/pull/3864

7 files changed, 55 insertions, 17 deletions

diff --git a/lib/strerror.c b/lib/strerror.c
index e273f3765..d0650d80c 100644
--- a/lib/strerror.c
+++ b/lib/strerror.c
@@ -311,6 +311,9 @@ curl_easy_strerror(CURLcode error)
   case CURLE_RECURSIVE_API_CALL:
     return "API function called from within callback";
 
+  case CURLE_AUTH_ERROR:
+    return "An authentication function returned an error";
+
     /* error codes not used by current libcurl */
   case CURLE_OBSOLETE20:
   case CURLE_OBSOLETE24:
diff --git a/lib/vauth/digest_sspi.c b/lib/vauth/digest_sspi.c
index fe8093e8b..850d6262b 100644
--- a/lib/vauth/digest_sspi.c
+++ b/lib/vauth/digest_sspi.c
@@ -220,7 +220,10 @@ CURLcode Curl_auth_create_digest_md5_message(struct Curl_easy *data,
     free(output_token);
     free(input_token);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Base64 encode the response */
@@ -607,7 +610,10 @@ CURLcode Curl_auth_create_digest_http_message(struct Curl_easy *data,
 
       Curl_safefree(digest->http_context);
 
-      return CURLE_OUT_OF_MEMORY;
+      if(status == SEC_E_INSUFFICIENT_MEMORY)
+        return CURLE_OUT_OF_MEMORY;
+
+      return CURLE_AUTH_ERROR;
     }
 
     output_token_len = resp_buf.cbBuffer;
diff --git a/lib/vauth/krb5_gssapi.c b/lib/vauth/krb5_gssapi.c
index ea0a5f189..95bab0e2e 100644
--- a/lib/vauth/krb5_gssapi.c
+++ b/lib/vauth/krb5_gssapi.c
@@ -121,7 +121,7 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
 
       free(spn);
 
-      return CURLE_OUT_OF_MEMORY;
+      return CURLE_AUTH_ERROR;
     }
 
     free(spn);
@@ -168,7 +168,7 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
     Curl_gss_log_error(data, "gss_init_sec_context() failed: ",
                        major_status, minor_status);
 
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
   }
 
   if(output_token.value && output_token.length) {
@@ -252,7 +252,7 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
 
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Convert the username from internal format to a displayable token */
@@ -264,7 +264,7 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
 
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Setup the challenge "input" security buffer */
@@ -355,7 +355,7 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
 
     free(message);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Base64 encode the response */
diff --git a/lib/vauth/krb5_sspi.c b/lib/vauth/krb5_sspi.c
index 1f6e462bf..6ac049eb3 100644
--- a/lib/vauth/krb5_sspi.c
+++ b/lib/vauth/krb5_sspi.c
@@ -217,8 +217,12 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
   /* Free the decoded challenge as it is not required anymore */
   free(chlg);
 
+  if(status == SEC_E_INSUFFICIENT_MEMORY) {
+    return CURLE_OUT_OF_MEMORY;
+  }
+
   if(status != SEC_E_OK && status != SEC_I_CONTINUE_NEEDED) {
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
   }
 
   if(memcmp(&context, krb5->context, sizeof(context))) {
@@ -309,7 +313,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
   if(status != SEC_E_OK) {
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Get the fully qualified username back from the context */
@@ -319,7 +326,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
   if(status != SEC_E_OK) {
     free(chlg);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Setup the "input" security buffer */
@@ -438,7 +448,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
     free(message);
     free(trailer);
 
-    return CURLE_OUT_OF_MEMORY;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Allocate the encryption (wrap) buffer */
diff --git a/lib/vauth/ntlm_sspi.c b/lib/vauth/ntlm_sspi.c
index 589cca16c..28109f76a 100644
--- a/lib/vauth/ntlm_sspi.c
+++ b/lib/vauth/ntlm_sspi.c
@@ -169,8 +169,10 @@ CURLcode Curl_auth_create_ntlm_type1_message(struct Curl_easy *data,
   if(status == SEC_I_COMPLETE_NEEDED ||
     status == SEC_I_COMPLETE_AND_CONTINUE)
     s_pSecFn->CompleteAuthToken(ntlm->context, &type_1_desc);
+  else if(status == SEC_E_INSUFFICIENT_MEMORY)
+    return CURLE_OUT_OF_MEMORY;
   else if(status != SEC_E_OK && status != SEC_I_CONTINUE_NEEDED)
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
 
   /* Base64 encode the response */
   return Curl_base64_encode(data, (char *) ntlm->output_token,
@@ -316,7 +318,10 @@ CURLcode Curl_auth_create_ntlm_type3_message(struct Curl_easy *data,
     infof(data, "NTLM handshake failure (type-3 message): Status=%x\n",
           status);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Base64 encode the response */
diff --git a/lib/vauth/spnego_gssapi.c b/lib/vauth/spnego_gssapi.c
index 5d43e1100..f05afca96 100644
--- a/lib/vauth/spnego_gssapi.c
+++ b/lib/vauth/spnego_gssapi.c
@@ -121,7 +121,7 @@ CURLcode Curl_auth_decode_spnego_message(struct Curl_easy *data,
 
       free(spn);
 
-      return CURLE_OUT_OF_MEMORY;
+      return CURLE_AUTH_ERROR;
     }
 
     free(spn);
@@ -177,7 +177,7 @@ CURLcode Curl_auth_decode_spnego_message(struct Curl_easy *data,
     if(output_token.value)
       gss_release_buffer(&unused_status, &output_token);
 
-    return CURLE_OUT_OF_MEMORY;
+    return CURLE_AUTH_ERROR;
   }
 
   /* Free previous token */
diff --git a/lib/vauth/spnego_sspi.c b/lib/vauth/spnego_sspi.c
index 4b21cc769..a4935276b 100644
--- a/lib/vauth/spnego_sspi.c
+++ b/lib/vauth/spnego_sspi.c
@@ -251,14 +251,25 @@ CURLcode Curl_auth_decode_spnego_message(struct Curl_easy *data,
     char buffer[STRERROR_LEN];
     failf(data, "InitializeSecurityContext failed: %s",
           Curl_sspi_strerror(nego->status, buffer, sizeof(buffer)));
-    return CURLE_OUT_OF_MEMORY;
+
+    if(nego->status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   if(nego->status == SEC_I_COMPLETE_NEEDED ||
      nego->status == SEC_I_COMPLETE_AND_CONTINUE) {
     nego->status = s_pSecFn->CompleteAuthToken(nego->context, &resp_desc);
     if(GSS_ERROR(nego->status)) {
-      return CURLE_RECV_ERROR;
+      char buffer[STRERROR_LEN];
+      failf(data, "CompleteAuthToken failed: %s",
+            Curl_sspi_strerror(nego->status, buffer, sizeof(buffer)));
+
+      if(nego->status == SEC_E_INSUFFICIENT_MEMORY)
+        return CURLE_OUT_OF_MEMORY;
+
+      return CURLE_AUTH_ERROR;
     }
   }