New authentication code added, particularly noticable when doing POST or PUT

with Digest or NTLM. libcurl will now use HEAD to negotiate the authentication
and when done perform the requested POST.

6 files changed, 92 insertions, 41 deletions

diff --git a/lib/file.c b/lib/file.c
index 72e835272..3977a73e5 100644
--- a/lib/file.c
+++ b/lib/file.c
@@ -196,7 +196,7 @@ CURLcode Curl_file(struct connectdata *conn)
   /* If we have selected NOBODY and HEADER, it means that we only want file
      information. Which for FILE can't be much more than the file size and
      date. */
-  if(data->set.no_body && data->set.include_header && fstated) {
+  if(conn->bits.no_body && data->set.include_header && fstated) {
     CURLcode result;
     sprintf(buf, "Content-Length: %" FORMAT_OFF_T "\r\n", expected_size);
     result = Curl_client_write(data, CLIENTWRITE_BOTH, buf, 0);
diff --git a/lib/ftp.c b/lib/ftp.c
index 6df8f22c6..dccfea3ff 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -1838,7 +1838,7 @@ CURLcode Curl_ftp_nextconnect(struct connectdata *conn)
       return result;
       
   }
-  else if(!data->set.no_body) {
+  else if(!conn->bits.no_body) {
     /* Retrieve file or directory */
     bool dirlist=FALSE;
     curl_off_t downloadsize=-1;
@@ -2209,7 +2209,7 @@ CURLcode ftp_perform(struct connectdata *conn,
   /* If we have selected NOBODY and HEADER, it means that we only want file
      information. Which in FTP can't be much more than the file size and
      date. */
-  if(data->set.no_body && data->set.include_header && ftp->file) {
+  if(conn->bits.no_body && data->set.include_header && ftp->file) {
     /* The SIZE command is _not_ RFC 959 specified, and therefor many servers
        may not support it! It is however the only way we have to get a file's
        size! */
@@ -2272,7 +2272,7 @@ CURLcode ftp_perform(struct connectdata *conn,
     return CURLE_OK;
   }
 
-  if(data->set.no_body)
+  if(conn->bits.no_body)
     /* doesn't really transfer any data */
     ftp->no_transfer = TRUE;
   /* Get us a second connection up and connected */
diff --git a/lib/http.c b/lib/http.c
index 41b1ef7c4..3c2eceff3 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -186,6 +186,17 @@ void Curl_http_auth_act(struct connectdata *conn)
       conn->newurl = strdup(data->change.url); /* clone URL */
     data->state.authavail = CURLAUTH_NONE; /* clear it here */
   }
+  else if(!data->state.authdone && (data->info.httpcode < 400)) {
+    /* no (known) authentication available,
+       authentication is not "done" yet and
+       no authentication seems to be required and
+       we didn't try HEAD or GET */
+    if((data->set.httpreq != HTTPREQ_GET) &&
+       (data->set.httpreq != HTTPREQ_HEAD)) {
+      conn->newurl = strdup(data->change.url); /* clone URL */
+      data->state.authdone = TRUE;
+    }
+  }
 }
 
 /**
@@ -204,13 +215,16 @@ static CURLcode http_auth_headers(struct connectdata *conn,
   char *auth=NULL;
 
   curlassert(data);
-  data->state.authdone = FALSE; /* default is no */
 
   if(!data->state.authstage) {
-    if(conn->bits.httpproxy && conn->bits.proxy_user_passwd)
+    if(conn->bits.httpproxy && conn->bits.proxy_user_passwd) {
+      data->state.authdone = FALSE;
       Curl_http_auth_stage(data, 407);
-    else if(conn->bits.user_passwd)
+    }
+    else if(conn->bits.user_passwd) {
+      data->state.authdone = FALSE;
       Curl_http_auth_stage(data, 401);
+    }
     else {
       data->state.authdone = TRUE;
       return CURLE_OK; /* no authentication with no user or password */
@@ -1139,6 +1153,7 @@ CURLcode Curl_http(struct connectdata *conn)
   const char *te = ""; /* tranfer-encoding */
   char *ptr;
   char *request;
+  Curl_HttpReq httpreq = data->set.httpreq;
 
   if(!conn->proto.http) {
     /* Only allocate this struct if we don't already have it! */
@@ -1157,19 +1172,40 @@ CURLcode Curl_http(struct connectdata *conn)
 
   if ( (conn->protocol&(PROT_HTTP|PROT_FTP)) &&
        data->set.upload) {
-    data->set.httpreq = HTTPREQ_PUT;
+    httpreq = HTTPREQ_PUT;
   }
 
-  request = data->set.customrequest?
-    data->set.customrequest:
-    (data->set.no_body?(char *)"HEAD":
-     ((HTTPREQ_POST == data->set.httpreq) ||
-      (HTTPREQ_POST_FORM == data->set.httpreq))?(char *)"POST":
-     (HTTPREQ_PUT == data->set.httpreq)?(char *)"PUT":(char *)"GET");
+  /* Now set the 'request' pointer to the proper request string */
+  if(data->set.customrequest)
+    request = data->set.customrequest;
+  else {
+    if(conn->bits.no_body)
+      request = (char *)"HEAD";
+    else {
+      curlassert((httpreq > HTTPREQ_NONE) && (httpreq < HTTPREQ_LAST));
+      switch(httpreq) {
+      case HTTPREQ_POST:
+      case HTTPREQ_POST_FORM:
+        request = (char *)"POST";
+        break;
+      case HTTPREQ_PUT:
+        request = (char *)"PUT";
+        break;
+      case HTTPREQ_GET:
+        request = (char *)"GET";
+        break;
+      case HTTPREQ_HEAD:
+        request = (char *)"HEAD";
+        break;
+      default: /* this should never happen */
+        break;
+      }
+    }
+  }
   
-  /* The User-Agent string has been built in url.c already, because it might
-     have been used in the proxy connect, but if we have got a header with
-     the user-agent string specified, we erase the previously made string
+  /* The User-Agent string might have been allocated in url.c already, because
+     it might have been used in the proxy connect, but if we have got a header
+     with the user-agent string specified, we erase the previously made string
      here. */
   if(checkheaders(data, "User-Agent:") && conn->allocptr.uagent) {
     free(conn->allocptr.uagent);
@@ -1181,6 +1217,16 @@ CURLcode Curl_http(struct connectdata *conn)
   if(result)
     return result;
 
+  if(!data->state.authdone && (httpreq != HTTPREQ_GET)) {
+    /* Until we are authenticated, we switch over to HEAD. Unless its a GET
+       we want to do. The explanation for this is rather long and boring, but
+       the point is that it can't be done otherwise without risking having to
+       send the POST or PUT data multiple times. */
+    httpreq = HTTPREQ_HEAD;
+    request = (char *)"HEAD";
+    conn->bits.no_body = TRUE;
+  }
+
   Curl_safefree(conn->allocptr.ref);
   if(data->change.referer && !checkheaders(data, "Referer:"))
     conn->allocptr.ref = aprintf("Referer: %s\015\012", data->change.referer);
@@ -1193,7 +1239,7 @@ CURLcode Curl_http(struct connectdata *conn)
   else
     conn->allocptr.cookie = NULL;
 
-  if(!conn->bits.upload_chunky && (data->set.httpreq != HTTPREQ_GET)) {
+  if(!conn->bits.upload_chunky && (httpreq != HTTPREQ_GET)) {
     /* not a chunky transfer yet, but data is to be sent */
     ptr = checkheaders(data, "Transfer-Encoding:");
     if(ptr) {
@@ -1284,7 +1330,7 @@ CURLcode Curl_http(struct connectdata *conn)
     /* The path sent to the proxy is in fact the entire URL */
     ppath = data->change.url;
   }
-  if(HTTPREQ_POST_FORM == data->set.httpreq) {
+  if(HTTPREQ_POST_FORM == httpreq) {
     /* we must build the whole darned post sequence first, so that we have
        a size of the whole shebang before we start to send it */
      result = Curl_getFormData(&http->sendit, data->set.httppost,
@@ -1303,9 +1349,9 @@ CURLcode Curl_http(struct connectdata *conn)
   if(!checkheaders(data, "Accept:"))
     http->p_accept = "Accept: */*\r\n";
 
-  if(( (HTTPREQ_POST == data->set.httpreq) ||
-       (HTTPREQ_POST_FORM == data->set.httpreq) ||
-       (HTTPREQ_PUT == data->set.httpreq) ) &&
+  if(( (HTTPREQ_POST == httpreq) ||
+       (HTTPREQ_POST_FORM == httpreq) ||
+       (HTTPREQ_PUT == httpreq) ) &&
      conn->resume_from) {
     /**********************************************************************
      * Resuming upload in HTTP means that we PUT or POST and that we have
@@ -1368,14 +1414,14 @@ CURLcode Curl_http(struct connectdata *conn)
      * or uploading and we always let customized headers override our internal
      * ones if any such are specified.
      */
-    if((data->set.httpreq == HTTPREQ_GET) &&
+    if((httpreq == HTTPREQ_GET) &&
        !checkheaders(data, "Range:")) {
       /* if a line like this was already allocated, free the previous one */
       if(conn->allocptr.rangeline)
         free(conn->allocptr.rangeline);
       conn->allocptr.rangeline = aprintf("Range: bytes=%s\r\n", conn->range);
     }
-    else if((data->set.httpreq != HTTPREQ_GET) &&
+    else if((httpreq != HTTPREQ_GET) &&
             !checkheaders(data, "Content-Range:")) {
 
       if(conn->resume_from) {
@@ -1538,11 +1584,11 @@ CURLcode Curl_http(struct connectdata *conn)
     http->postdata = NULL;  /* nothing to post at this point */
     Curl_pgrsSetUploadSize(data, 0); /* upload size is 0 atm */
 
-    /* If 'authdone' is still FALSE, we must not set the write socket index to
-       the Curl_transfer() call below, as we're not ready to actually upload
-       any data yet. */
+    /* If 'authdone' is FALSE, we must not set the write socket index to the
+       Curl_transfer() call below, as we're not ready to actually upload any
+       data yet. */
 
-    switch(data->set.httpreq) {
+    switch(httpreq) {
 
     case HTTPREQ_POST_FORM:
       if(Curl_FormInit(&http->form, http->sendit)) {
diff --git a/lib/transfer.c b/lib/transfer.c
index 4e5d71137..82c367978 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -498,7 +498,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  * If we requested a "no body", this is a good time to get
                  * out and return home.
                  */
-                if(data->set.no_body)
+                if(conn->bits.no_body)
                   stop_reading = TRUE;
                 else {
                   /* If we know the expected size of this document, we set the
@@ -555,10 +555,10 @@ CURLcode Curl_readwrite(struct connectdata *conn,
               /* This is the first header, it MUST be the error code line
                  or else we consiser this to be the body right away! */
               int httpversion_major;
-              int nc=sscanf (k->p, " HTTP/%d.%d %3d",
-                             &httpversion_major,
-                             &k->httpversion,
-                             &k->httpcode);
+              int nc=sscanf(k->p, " HTTP/%d.%d %3d",
+                            &httpversion_major,
+                            &k->httpversion,
+                            &k->httpcode);
               if (nc==3) {
                 k->httpversion += 10 * httpversion_major;
               }
@@ -566,7 +566,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                 /* this is the real world, not a Nirvana
                    NCSA 1.5.x returns this crap when asked for HTTP/1.1
                 */
-                nc=sscanf (k->p, " HTTP %3d", &k->httpcode);
+                nc=sscanf(k->p, " HTTP %3d", &k->httpcode);
                 k->httpversion = 10;
                
                /* If user has set option HTTP200ALIASES,
@@ -1274,7 +1274,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
      * returning.
      */
 
-    if(!(data->set.no_body) && (conn->size != -1) &&
+    if(!(conn->bits.no_body) && (conn->size != -1) &&
        (k->bytecount != conn->size) &&
        !conn->newurl) {
       failf(data, "transfer closed with %" FORMAT_OFF_T
@@ -1331,7 +1331,7 @@ CURLcode Curl_readwrite_init(struct connectdata *conn)
       Curl_pgrsSetDownloadSize(data, conn->size);
   }
   /* we want header and/or body, if neither then don't do this! */
-  if(conn->bits.getheader || !data->set.no_body) {
+  if(conn->bits.getheader || !conn->bits.no_body) {
 
     FD_ZERO (&k->readfd);               /* clear it */
     if(conn->sockfd != CURL_SOCKET_BAD) {
@@ -1420,7 +1420,6 @@ void Curl_single_fdset(struct connectdata *conn,
 static CURLcode
 Transfer(struct connectdata *conn)
 {
-  struct SessionHandle *data = conn->data;
   CURLcode result;
   struct Curl_transfer_keeper *k = &conn->keep;
   bool done=FALSE;
@@ -1435,7 +1434,7 @@ Transfer(struct connectdata *conn)
     return CURLE_OK;
 
   /* we want header and/or body, if neither then don't do this! */
-  if(!conn->bits.getheader && data->set.no_body)
+  if(!conn->bits.getheader && conn->bits.no_body)
     return CURLE_OK;
 
   k->writefdp = &k->writefd; /* store the address of the set */
@@ -1859,7 +1858,7 @@ CURLcode Curl_follow(struct SessionHandle *data,
     if(data->set.httpreq != HTTPREQ_GET) {
       data->set.httpreq = HTTPREQ_GET; /* enforce GET request */
       infof(data, "Disables POST, goes with %s\n",
-            data->set.no_body?"HEAD":"GET");
+            data->set.opt_no_body?"HEAD":"GET");
     }
     break;
   case 304: /* Not Modified */
diff --git a/lib/url.c b/lib/url.c
index 805acd0e6..40794a93e 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -463,7 +463,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, ...)
     /*
      * Do not include the body part in the output data stream.
      */
-    data->set.no_body = va_arg(param, long)?TRUE:FALSE;
+    data->set.opt_no_body = va_arg(param, long)?TRUE:FALSE;
     break;
   case CURLOPT_FAILONERROR:
     /*
@@ -2066,6 +2066,7 @@ static CURLcode CreateConnection(struct SessionHandle *data,
 
   conn->bits.user_passwd = data->set.userpwd?1:0;
   conn->bits.proxy_user_passwd = data->set.proxyuserpwd?1:0;
+  conn->bits.no_body = data->set.opt_no_body;
 
   /* This initing continues below, see the comment "Continue connectdata
    * initialization here" */
@@ -2924,6 +2925,9 @@ static CURLcode CreateConnection(struct SessionHandle *data,
     conn->bits.user_passwd = old_conn->bits.user_passwd;
     conn->bits.proxy_user_passwd = old_conn->bits.proxy_user_passwd;
 
+    /* get the newly set value, not the old one */
+    conn->bits.no_body = old_conn->bits.no_body;
+
     /* If we speak over a proxy, we need to copy the host name too, as it
        might be another remote host even when re-using a connection */
     strcpy(conn->gname, old_conn->gname); /* safe strcpy() */
diff --git a/lib/urldata.h b/lib/urldata.h
index 196c45f16..4ed66243d 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -302,6 +302,7 @@ struct ConnectBits {
                          call */
   bool retry;         /* this connection is about to get closed and then
                          re-attempted at another connection. */
+  bool no_body;       /* CURLOPT_NO_BODY (or similar) was set */
 };
 
 /*
@@ -644,6 +645,7 @@ typedef enum {
   HTTPREQ_POST,
   HTTPREQ_POST_FORM, /* we make a difference internally */
   HTTPREQ_PUT,
+  HTTPREQ_HEAD,
   HTTPREQ_CUSTOM,
   HTTPREQ_LAST /* last in list */
 } Curl_HttpReq;
@@ -866,7 +868,7 @@ struct UserDefined {
 
   bool http_set_referer;
   bool http_auto_referer; /* set "correct" referer when following location: */
-  bool no_body;
+  bool opt_no_body;      /* as set with CURLOPT_NO_BODY */
   bool set_port;
   bool upload;
   enum CURL_NETRC_OPTION