vauth: Use CURLE_AUTH_ERROR for auth function errors

- Add new error code CURLE_AUTH_ERROR.

Prior to this change auth function errors were signaled by
CURLE_OUT_OF_MEMORY and CURLE_RECV_ERROR, and neither one was
technically correct.

Ref: https://github.com/curl/curl/pull/3848

Co-authored-by: Dominik Hölzl

Closes https://github.com/curl/curl/pull/3864

1 files changed, 17 insertions, 4 deletions

diff --git a/lib/vauth/krb5_sspi.c b/lib/vauth/krb5_sspi.c
index 1f6e462bf..6ac049eb3 100644
--- a/lib/vauth/krb5_sspi.c
+++ b/lib/vauth/krb5_sspi.c
@@ -217,8 +217,12 @@ CURLcode Curl_auth_create_gssapi_user_message(struct Curl_easy *data,
   /* Free the decoded challenge as it is not required anymore */
   free(chlg);
 
+  if(status == SEC_E_INSUFFICIENT_MEMORY) {
+    return CURLE_OUT_OF_MEMORY;
+  }
+
   if(status != SEC_E_OK && status != SEC_I_CONTINUE_NEEDED) {
-    return CURLE_RECV_ERROR;
+    return CURLE_AUTH_ERROR;
   }
 
   if(memcmp(&context, krb5->context, sizeof(context))) {
@@ -309,7 +313,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
   if(status != SEC_E_OK) {
     free(chlg);
 
-    return CURLE_OUT_OF_MEMORY;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Get the fully qualified username back from the context */
@@ -319,7 +326,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
   if(status != SEC_E_OK) {
     free(chlg);
 
-    return CURLE_RECV_ERROR;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Setup the "input" security buffer */
@@ -438,7 +448,10 @@ CURLcode Curl_auth_create_gssapi_security_message(struct Curl_easy *data,
     free(message);
     free(trailer);
 
-    return CURLE_OUT_OF_MEMORY;
+    if(status == SEC_E_INSUFFICIENT_MEMORY)
+      return CURLE_OUT_OF_MEMORY;
+
+    return CURLE_AUTH_ERROR;
   }
 
   /* Allocate the encryption (wrap) buffer */