Introducing curl_easy_pause() and new magic return codes for both the read

and the write callbacks that now can make a connection's reading and/or
writing get paused.

6 files changed, 231 insertions, 27 deletions

diff --git a/lib/easy.c b/lib/easy.c
index adc88ec2f..991a25573 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -744,6 +744,107 @@ void curl_easy_reset(CURL *curl)
   data->set.new_directory_perms = 0755; /* Default permissions */
 }
 
+/*
+ * curl_easy_pause() allows an application to pause or unpause a specific
+ * transfer and direction. This function sets the full new state for the
+ * current connection this easy handle operates on.
+ *
+ * NOTE: if you have the receiving paused and you call this function to remove
+ * the pausing, you may get your write callback called at this point.
+ *
+ * Action is a bitmask consisting of CURLPAUSE_* bits in curl/curl.h
+ */
+CURLcode curl_easy_pause(CURL *curl, int action)
+{
+  struct SessionHandle *data = (struct SessionHandle *)curl;
+  struct SingleRequest *k = &data->req;
+  CURLcode result = CURLE_OK;
+
+  /* first switch off both pause bits */
+  int newstate = k->keepon &~ (KEEP_READ_PAUSE| KEEP_WRITE_PAUSE);
+
+  /* set the new desired pause bits */
+  newstate |= ((action & CURLPAUSE_RECV)?KEEP_READ_PAUSE:0) |
+    ((action & CURLPAUSE_SEND)?KEEP_WRITE_PAUSE:0);
+
+  /* put it back in the keepon */
+  k->keepon = newstate;
+
+  if(!(newstate & KEEP_READ_PAUSE) && data->state.tempwrite) {
+    /* we have a buffer for writing that we now seem to be able to deliver since
+       the receive pausing is lifted! */
+
+    /* get the pointer, type and length in local copies since the function may
+       return PAUSE again and then we'll get a new copy allocted and stored in
+       the tempwrite variables */
+    char *tempwrite = data->state.tempwrite;
+    size_t tempsize = data->state.tempwritesize;
+    int temptype = data->state.tempwritetype;
+    size_t chunklen;
+
+    /* clear tempwrite here just to make sure it gets cleared if there's no
+       further use of it, and make sure we don't clear it after the function
+       invoke as it may have been set to a new value by then */
+    data->state.tempwrite = NULL;
+
+    /* since the write callback API is define to never exceed
+       CURL_MAX_WRITE_SIZE bytes in a single call, and since we may in fact
+       have more data than that in our buffer here, we must loop sending the
+       data in multiple calls until there's no data left or we get another
+       pause returned.
+
+       A tricky part is that the function we call will "buffer" the data
+       itself when it pauses on a particular buffer, so we may need to do some
+       extra trickery if we get a pause return here.
+    */
+    do {
+      chunklen = (tempsize > CURL_MAX_WRITE_SIZE)?CURL_MAX_WRITE_SIZE:tempsize;
+
+      result = Curl_client_write(data->state.current_conn,
+                                 temptype, tempwrite, chunklen);
+      if(!result)
+        /* failures abort the loop at once */
+        break;
+
+      if(data->state.tempwrite && (tempsize - chunklen)) {
+        /* Ouch, the reading is again paused and the block we send is now
+           "cached". If this is the final chunk we can leave it like this, but
+           if we have more chunks that is cached after this, we need to free
+           the newly cached one and put back a version that is truly the entire
+           contents that is saved for later
+        */
+        char *newptr;
+
+        free(data->state.tempwrite); /* free the one just cached as it isn't
+                                        enough */
+
+        /* note that tempsize is still the size as before the callback was
+           used, and thus the whole piece of data to keep */
+        newptr = malloc(tempsize);
+        if(!newptr) {
+          result = CURLE_OUT_OF_MEMORY;
+          /* tempwrite will be freed further down */
+          break;
+        }
+        data->state.tempwrite = newptr; /* store new pointer */
+        memcpy(newptr, tempwrite, tempsize);
+        data->state.tempwritesize = tempsize; /* store new size */
+        /* tempwrite will be freed further down */
+        break; /* go back to pausing until further notice */
+      }
+      else {
+        tempsize -= chunklen;  /* left after the call above */
+        tempwrite += chunklen; /* advance the pointer */
+      }
+
+    } while((result == CURLE_OK) && tempsize);
+
+    free(tempwrite); /* this is unconditionally no longer used */
+  }
+
+  return result;
+}
+
 #ifdef CURL_DOES_CONVERSIONS
 /*
  * Curl_convert_to_network() is an internal function
diff --git a/lib/multi.c b/lib/multi.c
index 042694ca5..a88a0b74a 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -1255,13 +1255,13 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       k = &easy->easy_handle->req;
 
       if(!(k->keepon & KEEP_READ)) {
-          /* We're done reading */
-          easy->easy_conn->readchannel_inuse = FALSE;
+        /* We're done reading */
+        easy->easy_conn->readchannel_inuse = FALSE;
       }
 
       if(!(k->keepon & KEEP_WRITE)) {
-          /* We're done writing */
-          easy->easy_conn->writechannel_inuse = FALSE;
+        /* We're done writing */
+        easy->easy_conn->writechannel_inuse = FALSE;
       }
 
       if(easy->result)  {
diff --git a/lib/sendf.c b/lib/sendf.c
index b33277ac3..32a8655b2 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -376,6 +376,36 @@ CURLcode Curl_write(struct connectdata *conn,
   return retcode;
 }
 
+static CURLcode pausewrite(struct SessionHandle *data,
+                           int type, /* what type of data */
+                           char *ptr,
+                           size_t len)
+{
+  /* signalled to pause sending on this connection, but since we have data
+     we want to send we need to dup it to save a copy for when the sending
+     is again enabled */
+  struct SingleRequest *k = &data->req;
+  char *dupl = malloc(len);
+  if(!dupl)
+    return CURLE_OUT_OF_MEMORY;
+
+  memcpy(dupl, ptr, len);
+
+  /* store this information in the state struct for later use */
+  data->state.tempwrite = dupl;
+  data->state.tempwritesize = len;
+  data->state.tempwritetype = type;
+
+  /* mark the connection as RECV paused */
+  k->keepon |= KEEP_READ_PAUSE;
+
+  DEBUGF(infof(data, "Pausing with %d bytes in buffer for type %02x\n",
+               (int)len, type));
+
+  return CURLE_OK;
+}
+
+
 /* client_write() sends data to the write callback(s)
 
    The bit pattern defines to what "streams" to write to. Body and/or header.
@@ -390,8 +420,37 @@ CURLcode Curl_client_write(struct connectdata *conn,
   size_t wrote;
 
   if(data->state.cancelled) {
-      /* We just suck everything into a black hole */
-      return CURLE_OK;
+    /* We just suck everything into a black hole */
+    return CURLE_OK;
+  }
+
+  /* If reading is actually paused, we're forced to append this chunk of data
+     to the already held data, but only if it is the same type as otherwise it
+     can't work and it'll return error instead. */
+  if(data->req.keepon & KEEP_READ_PAUSE) {
+    size_t newlen;
+    char *newptr;
+    if(type != data->state.tempwritetype)
+      /* major internal confusion */
+      return CURLE_RECV_ERROR;
+
+    /* figure out the new size of the data to save */
+    newlen = len + data->state.tempwritesize;
+    /* allocate the new memory area */
+    newptr = malloc(newlen);
+    if(!newptr)
+      return CURLE_OUT_OF_MEMORY;
+    /* copy the previously held data to the new area */
+    memcpy(newptr, data->state.tempwrite, data->state.tempwritesize);
+    /* copy the new data to the end of the new area */
+    memcpy(newptr + data->state.tempwritesize, ptr, len);
+    /* free the old data */
+    free(data->state.tempwrite);
+    /* update the pointer and the size */
+    data->state.tempwrite = newptr;
+    data->state.tempwritesize = newlen;
+
+    return CURLE_OK;
   }
 
   if(0 == len)
@@ -422,8 +481,11 @@ CURLcode Curl_client_write(struct connectdata *conn,
       wrote = len;
     }
 
+    if(CURL_WRITEFUNC_PAUSE == wrote)
+      return pausewrite(data, type, ptr, len);
+
     if(wrote != len) {
-      failf (data, "Failed writing body");
+      failf(data, "Failed writing body (%d != %d)", (int)wrote, (int)len);
       return CURLE_WRITE_ERROR;
     }
   }
@@ -441,6 +503,12 @@ CURLcode Curl_client_write(struct connectdata *conn,
        regardless of the ftp transfer mode (ASCII/Image) */
 
     wrote = writeit(ptr, 1, len, data->set.writeheader);
+    if(CURL_WRITEFUNC_PAUSE == wrote)
+      /* here we pass in the HEADER bit only since if this was body as well
+         then it was passed already and clearly that didn't trigger the pause,
+         so this is saved for later with the HEADER bit only */
+      return pausewrite(data, CLIENTWRITE_HEADER, ptr, len);
+
     if(wrote != len) {
       failf (data, "Failed writing header");
       return CURLE_WRITE_ERROR;
diff --git a/lib/transfer.c b/lib/transfer.c
index 0817fa024..5b391318a 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -134,6 +134,14 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
     failf(data, "operation aborted by callback\n");
     return CURLE_ABORTED_BY_CALLBACK;
   }
+  else if(nread == CURL_READFUNC_PAUSE) {
+    struct SingleRequest *k = &data->req;
+    k->keepon |= KEEP_READ_PAUSE; /* mark reading as paused */
+    return 0; /* nothing was read */
+  }
+  else if((size_t)nread > buffersize)
+    /* the read function returned a too large value */
+    return CURLE_READ_ERROR;
 
   if(!conn->bits.forbidchunk && conn->bits.upload_chunky) {
     /* if chunked Transfer-Encoding */
@@ -330,7 +338,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
   /* only use the proper socket if the *_HOLD bit is not set simultaneously as
      then we are in rate limiting state in that transfer direction */
 
-  if((k->keepon & (KEEP_READ|KEEP_READ_HOLD)) == KEEP_READ) {
+  if((k->keepon & KEEP_READBITS) == KEEP_READ) {
     fd_read = conn->sockfd;
 #if defined(USE_LIBSSH2)
     if(conn->protocol & (PROT_SCP|PROT_SFTP))
@@ -339,7 +347,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
   } else
     fd_read = CURL_SOCKET_BAD;
 
-  if((k->keepon & (KEEP_WRITE|KEEP_WRITE_HOLD)) == KEEP_WRITE)
+  if((k->keepon & KEEP_WRITEBITS) == KEEP_WRITE)
     fd_write = conn->writesockfd;
   else
     fd_write = CURL_SOCKET_BAD;
@@ -1425,9 +1433,11 @@ CURLcode Curl_readwrite(struct connectdata *conn,
           else
             nread = 0; /* we're done uploading/reading */
 
-          /* the signed int typecase of nread of for systems that has
-             unsigned size_t */
-          if(nread<=0) {
+          if(!nread && (k->keepon & KEEP_READ_PAUSE)) {
+            /* this is a paused transfer */
+            break;
+          }
+          else if(nread<=0) {
             /* done */
             k->keepon &= ~KEEP_WRITE; /* we're done writing */
             writedone = TRUE;
@@ -1635,7 +1645,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
   }
 
   /* Now update the "done" boolean we return */
-  *done = (bool)(0 == (k->keepon&(KEEP_READ|KEEP_WRITE)));
+  *done = (bool)(0 == (k->keepon&(KEEP_READ|KEEP_WRITE|KEEP_READ_PAUSE|KEEP_WRITE_PAUSE)));
 
   return CURLE_OK;
 }
@@ -1660,7 +1670,8 @@ int Curl_single_getsock(const struct connectdata *conn,
     /* simple check but we might need two slots */
     return GETSOCK_BLANK;
 
-  if(data->req.keepon & KEEP_READ) {
+  /* don't include HOLD and PAUSE connections */
+  if((data->req.keepon & KEEP_READBITS) == KEEP_READ) {
 
     DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
 
@@ -1668,7 +1679,8 @@ int Curl_single_getsock(const struct connectdata *conn,
     sock[sockindex] = conn->sockfd;
   }
 
-  if(data->req.keepon & KEEP_WRITE) {
+  /* don't include HOLD and PAUSE connections */
+  if((data->req.keepon & KEEP_WRITEBITS) == KEEP_WRITE) {
 
     if((conn->sockfd != conn->writesockfd) ||
        !(data->req.keepon & KEEP_READ)) {
@@ -1751,10 +1763,17 @@ Transfer(struct connectdata *conn)
         k->keepon |= KEEP_READ_HOLD; /* hold it */
     }
 
-    /* The *_HOLD logic is necessary since even though there might be no
-       traffic during the select interval, we still call Curl_readwrite() for
-       the timeout case and if we limit transfer speed we must make sure that
-       this function doesn't transfer anything while in HOLD status. */
+    /* pause logic. Don't check descriptors for paused connections */
+    if(k->keepon & KEEP_READ_PAUSE)
+      fd_read = CURL_SOCKET_BAD;
+    if(k->keepon & KEEP_WRITE_PAUSE)
+      fd_write = CURL_SOCKET_BAD;
+
+    /* The *_HOLD and *_PAUSE logic is necessary since even though there might
+       be no traffic during the select interval, we still call
+       Curl_readwrite() for the timeout case and if we limit transfer speed we
+       must make sure that this function doesn't transfer anything while in
+       HOLD status. */
 
     switch (Curl_socket_ready(fd_read, fd_write, 1000)) {
     case -1: /* select() error, stop reading */
diff --git a/lib/url.c b/lib/url.c
index 25d40b922..d7b4c512a 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -4433,6 +4433,13 @@ CURLcode Curl_done(struct connectdata **connp,
 
   Curl_pgrsDone(conn); /* done with the operation */
 
+  /* if the transfer was completed in a paused state there can be buffered
+     data left to write and then kill */
+  if(data->state.tempwrite) {
+    free(data->state.tempwrite);
+    data->state.tempwrite = NULL;
+  }
+
   /* for ares-using, make sure all possible outstanding requests are properly
      cancelled before we proceed */
   ares_cancel(data->state.areschannel);
diff --git a/lib/urldata.h b/lib/urldata.h
index 5b5c014b4..976bc3771 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -629,12 +629,18 @@ struct hostname {
  */
 
 #define KEEP_NONE  0
-#define KEEP_READ  1      /* there is or may be data to read */
-#define KEEP_WRITE 2      /* there is or may be data to write */
-#define KEEP_READ_HOLD 4  /* when set, no reading should be done but there
-                             might still be data to read */
-#define KEEP_WRITE_HOLD 8 /* when set, no writing should be done but there
-                             might still be data to write */
+#define KEEP_READ  (1<<0)     /* there is or may be data to read */
+#define KEEP_WRITE (1<<1)     /* there is or may be data to write */
+#define KEEP_READ_HOLD (1<<2) /* when set, no reading should be done but there
+                                 might still be data to read */
+#define KEEP_WRITE_HOLD (1<<3) /* when set, no writing should be done but there
+                                  might still be data to write */
+#define KEEP_READ_PAUSE (1<<4) /* reading is paused */
+#define KEEP_WRITE_PAUSE (1<<5) /* writing is paused */
+
+#define KEEP_READBITS (KEEP_READ | KEEP_READ_HOLD | KEEP_READ_PAUSE)
+#define KEEP_WRITEBITS (KEEP_WRITE | KEEP_WRITE_HOLD | KEEP_WRITE_PAUSE)
+
 
 #ifdef HAVE_LIBZ
 typedef enum {
@@ -1126,10 +1132,13 @@ struct UrlState {
                        following not keep sending user+password... This is
                        strdup() data.
                     */
-
   struct curl_ssl_session *session; /* array of 'numsessions' size */
   long sessionage;                  /* number of the most recent session */
-
+  char *tempwrite;      /* allocated buffer to keep data in when a write
+                           callback returns to make the connection paused */
+  size_t tempwritesize; /* size of the 'tempwrite' allocated buffer */
+  int tempwritetype;    /* type of the 'tempwrite' buffer as a bitmask that is
+                           used with Curl_client_write() */
   char *scratch; /* huge buffer[BUFSIZE*2] when doing upload CRLF replacing */
   bool errorbuf; /* Set to TRUE if the error buffer is already filled in.
                     This must be set to FALSE every time _easy_perform() is