The initial HTTP request can now be sent in multiple parts, as part of the

regular transfer process. This required some new tweaks, like for example
we need to be able to tell the tranfer loop to not chunky-encode uploads
while we're transferring the rest of the request...

3 files changed, 171 insertions, 76 deletions

diff --git a/lib/http.c b/lib/http.c
index f9f0a819b..49ba70f36 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -98,12 +98,65 @@
 #include "memdebug.h"
 #endif
 
+/* fread() emulation to provide POST and/or request data */
+static int readmoredata(char *buffer,
+                        size_t size,
+                        size_t nitems,
+                        void *userp)
+{
+  struct connectdata *conn = (struct connectdata *)userp;
+  struct HTTP *http = conn->proto.http;
+  int fullsize = size * nitems;
+
+  if(0 == http->postsize)
+    /* nothing to return */
+    return 0;
+  
+  /* make sure that a HTTP request is never sent away chunked! */
+  conn->bits.forbidchunk= (http->sending == HTTPSEND_REQUEST)?TRUE:FALSE;
+
+  if(http->postsize <= fullsize) {
+    memcpy(buffer, http->postdata, http->postsize);
+    fullsize = http->postsize;
+
+    if(http->backup.postsize) {
+      /* move backup data into focus and continue on that */
+      http->postdata = http->backup.postdata;
+      http->postsize = http->backup.postsize;
+      conn->fread =    http->backup.fread;
+      conn->fread_in = http->backup.fread_in;
+
+      http->sending++; /* move one step up */
+
+      http->backup.postsize=0;
+    }
+    else
+      http->postsize = 0;
+
+    return fullsize;
+  }
+
+  memcpy(buffer, http->postdata, fullsize);
+  http->postdata += fullsize;
+  http->postsize -= fullsize;
+
+  return fullsize;
+}
+
 /* ------------------------------------------------------------------------- */
 /*
  * The add_buffer series of functions are used to build one large memory chunk
  * from repeated function invokes. Used so that the entire HTTP request can
  * be sent in one go.
  */
+
+struct send_buffer {
+  char *buffer;
+  size_t size_max;
+  size_t size_used;
+};
+typedef struct send_buffer send_buffer;
+
 static CURLcode
  add_buffer(send_buffer *in, const void *inptr, size_t size);
 
@@ -136,33 +189,52 @@ CURLcode add_buffer_send(send_buffer *in,
   CURLcode res;
   char *ptr;
   int size;
+  struct HTTP *http = conn->proto.http;
 
   /* The looping below is required since we use non-blocking sockets, but due
      to the circumstances we will just loop and try again and again etc */
 
   ptr = in->buffer;
   size = in->size_used;
-  do {
-    res = Curl_write(conn, sockfd, ptr, size, &amount);
 
-    if(CURLE_OK != res)
-      break;
+  res = Curl_write(conn, sockfd, ptr, size, &amount);
+
+  if(CURLE_OK == res) {
 
     if(conn->data->set.verbose)
       /* this data _may_ contain binary stuff */
       Curl_debug(conn->data, CURLINFO_HEADER_OUT, ptr, amount);
 
     *bytes_written += amount;
-
+    
     if(amount != size) {
+      /* The whole request could not be sent in one system call. We must queue
+         it up and send it later when we get the chance. We must not loop here
+         and wait until it might work again. */
+
       size -= amount;
       ptr += amount;
+    
+      /* backup the currently set pointers */
+      http->backup.fread = conn->fread;
+      http->backup.fread_in = conn->fread_in;
+      http->backup.postdata = http->postdata;
+      http->backup.postsize = http->postsize;
+
+      /* set the new pointers for the request-sending */
+      conn->fread = (curl_read_callback)readmoredata;
+      conn->fread_in = (void *)conn;
+      http->postdata = ptr;
+      http->postsize = size;
+
+      http->send_buffer = in;
+      http->sending = HTTPSEND_REQUEST;
+      
+      return CURLE_OK;
     }
-    else
-      break;
-
-  } while(1);
 
+    /* the full buffer was sent, clean up and return */
+  }
   if(in->buffer)
     free(in->buffer);
   free(in);
@@ -519,6 +591,13 @@ CURLcode Curl_http_done(struct connectdata *conn)
   conn->fread = data->set.fread; /* restore */
   conn->fread_in = data->set.in; /* restore */
 
+  if(http->send_buffer) {
+    send_buffer *buff = http->send_buffer;
+    
+    free(buff->buffer);
+    free(buff);
+  }
+
   if(HTTPREQ_POST_FORM == data->set.httpreq) {
     conn->bytecount = http->readbytecount + http->writebytecount;
       
@@ -537,33 +616,6 @@ CURLcode Curl_http_done(struct connectdata *conn)
   return CURLE_OK;
 }
 
-/* fread() emulation to provide POST data */
-static int POSTReader(char *buffer,
-                      size_t size,
-                      size_t nitems,
-                      void *userp)
-{
-  struct HTTP *http = (struct HTTP *)userp;
-  int fullsize = size * nitems;
-
-  if(0 == http->postsize)
-    /* nothing to return */
-    return 0;
-  
-  if(http->postsize <= fullsize) {
-    memcpy(buffer, http->postdata, http->postsize);
-    fullsize = http->postsize;
-    http->postsize = 0;
-    return fullsize;
-  }
-
-  memcpy(buffer, http->postdata, fullsize);
-  http->postdata += fullsize;
-  http->postsize -= fullsize;
-
-  return fullsize;
-}
-
 CURLcode Curl_http(struct connectdata *conn)
 {
   struct SessionHandle *data=conn->data;
@@ -957,6 +1009,8 @@ CURLcode Curl_http(struct connectdata *conn)
       conn->fread = (curl_read_callback)Curl_FormReader;
       conn->fread_in = &http->form;
 
+      http->sending = HTTPSEND_BODY;
+
       if(!conn->bits.upload_chunky)
         /* only add Content-Length if not uploading chunked */
         add_bufferf(req_buffer,
@@ -1076,9 +1130,17 @@ CURLcode Curl_http(struct connectdata *conn)
           http->postsize = strlen(data->set.postfields);
         http->postdata = data->set.postfields;
 
-        conn->fread = (curl_read_callback)POSTReader;
-        conn->fread_in = (void *)http;
+        http->sending = HTTPSEND_BODY;
+
+        conn->fread = (curl_read_callback)readmoredata;
+        conn->fread_in = (void *)conn;
+
+        /* set the upload size to the progress meter */
+        Curl_pgrsSetUploadSize(data, http->postsize);
       }
+      else
+        /* set the upload size to the progress meter */
+        Curl_pgrsSetUploadSize(data, data->set.infilesize);
 
       /* issue the request, headers-only */
       result = add_buffer_send(req_buffer, conn->firstsocket, conn,
@@ -1107,7 +1169,8 @@ CURLcode Curl_http(struct connectdata *conn)
         /* HTTP GET/HEAD download: */
         result = Curl_Transfer(conn, conn->firstsocket, -1, TRUE,
                                &http->readbytecount,
-                               -1, NULL); /* nothing to upload */
+                               http->postdata?conn->firstsocket:-1,
+                               http->postdata?&http->writebytecount:NULL);
     }
     if(result)
       return result;
diff --git a/lib/transfer.c b/lib/transfer.c
index 16c1a7b21..db59ca5ad 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -118,6 +118,51 @@ enum {
    changed. It should just remain a blanked-out timeout value. */
 static struct timeval notimeout={0,0};
 
+/*
+ * This function will call the read callback to fill our buffer with data
+ * to upload.
+ */
+static int fillbuffer(struct connectdata *conn,
+                      int bytes)
+{
+  int buffersize = bytes;
+  int nread;
+
+  if(conn->bits.upload_chunky) {
+    /* if chunked Transfer-Encoding */
+    buffersize -= (8 + 2 + 2);   /* 32bit hex + CRLF + CRLF */
+    conn->upload_fromhere += 10; /* 32bit hex + CRLF */
+  }
+  
+  nread = conn->fread(conn->upload_fromhere, 1,
+                      buffersize, conn->fread_in);
+          
+  if(!conn->bits.forbidchunk && conn->bits.upload_chunky) {
+    /* if chunked Transfer-Encoding */
+    char hexbuffer[11];
+    int hexlen = snprintf(hexbuffer, sizeof(hexbuffer),
+                          "%x\r\n", nread);
+    /* move buffer pointer */
+    conn->upload_fromhere -= hexlen;
+    nread += hexlen;
+
+    /* copy the prefix to the buffer */
+    memcpy(conn->upload_fromhere, hexbuffer, hexlen);
+    if(nread>hexlen) {
+      /* append CRLF to the data */
+      memcpy(conn->upload_fromhere +
+             nread, "\r\n", 2);
+      nread+=2;
+    }
+    else {
+      /* mark this as done once this chunk is transfered */
+      conn->keep.upload_done = TRUE;
+    }
+  }
+  return nread;
+}
+
+
 CURLcode Curl_readwrite(struct connectdata *conn,
                         bool *done)
 {
@@ -862,44 +907,11 @@ CURLcode Curl_readwrite(struct connectdata *conn,
         /* only read more data if there's no upload data already
            present in the upload buffer */
         if(0 == conn->upload_present) {
-          size_t buffersize = BUFSIZE;
           /* init the "upload from here" pointer */
           conn->upload_fromhere = k->uploadbuf;
 
-          if(!k->upload_done) {
-
-            if(conn->bits.upload_chunky) {
-              /* if chunked Transfer-Encoding */
-              buffersize -= (8 + 2 + 2);   /* 32bit hex + CRLF + CRLF */
-              conn->upload_fromhere += 10; /* 32bit hex + CRLF */
-            }
-
-            nread = conn->fread(conn->upload_fromhere, 1,
-                                buffersize, conn->fread_in);
-          
-            if(conn->bits.upload_chunky) {
-              /* if chunked Transfer-Encoding */
-              char hexbuffer[9];
-              int hexlen = snprintf(hexbuffer, sizeof(hexbuffer),
-                                    "%x\r\n", nread);
-              /* move buffer pointer */
-              conn->upload_fromhere -= hexlen;
-              nread += hexlen;
-
-              /* copy the prefix to the buffer */
-              memcpy(conn->upload_fromhere, hexbuffer, hexlen);
-              if(nread>hexlen) {
-                /* append CRLF to the data */
-                memcpy(conn->upload_fromhere +
-                       nread, "\r\n", 2);
-                nread+=2;
-              }
-              else {
-                /* mark this as done once this chunk is transfered */
-                k->upload_done = TRUE;
-              }
-            }
-          }
+          if(!k->upload_done)
+            nread = fillbuffer(conn, BUFSIZE);
           else
             nread = 0; /* we're done uploading/reading */
 
diff --git a/lib/urldata.h b/lib/urldata.h
index 6e66d2483..0a54b0ee4 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -157,6 +157,8 @@ struct ssl_config_data {
 struct HTTP {
   struct FormData *sendit;
   int postsize;
+  char *postdata;
+
   const char *p_pragma;      /* Pragma: string */
   const char *p_accept;      /* Accept: string */
   long readbytecount; 
@@ -166,7 +168,22 @@ struct HTTP {
   struct Form form;
   struct Curl_chunker chunk;
 
-  char *postdata; /* for regular POSTs */
+  struct back {
+    curl_read_callback fread; /* backup storage for fread pointer */
+    void *fread_in;           /* backup storage for fread_in pointer */
+    char *postdata;
+    int postsize;
+  } backup;
+
+  enum {
+    HTTPSEND_NADA,    /* init */
+    HTTPSEND_REQUEST, /* sending a request */
+    HTTPSEND_BODY,    /* sending body */
+    HTTPSEND_LAST     /* never use this */
+  } sending;
+
+  void *send_buffer; /* used if the request couldn't be sent in one chunk,
+                        points to an allocated send_buffer struct */
 };
 
 /****************************************************************************
@@ -221,8 +238,11 @@ struct ConnectBits {
 
   bool upload_chunky; /* set TRUE if we are doing chunked transfer-encoding
                          on upload */
+  bool getheader;     /* TRUE if header parsing is wanted */
 
-  bool getheader;	 /* TRUE if header parsing is wanted */
+  bool forbidchunk;   /* used only to explicitly forbid chunk-upload for
+                         specific upload buffers. See readmoredata() in
+                         http.c for details. */
 };
 
 /*