pause: handle mixed types of data when paused

When receiving chunked encoded data with trailers, and the write
callback returns PAUSE, there might be both body and header to store to
resend on unpause. Previously libcurl returned error for that case.

Added test case 1540 to verify.

Reported-by: Stephen Toub
Fixes #1354
Closes #1357

8 files changed, 292 insertions, 56 deletions

diff --git a/lib/easy.c b/lib/easy.c
index cf65af911..b3159d1a2 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -1011,19 +1011,32 @@ CURLcode curl_easy_pause(struct Curl_easy *data, int action)
   /* put it back in the keepon */
   k->keepon = newstate;
 
-  if(!(newstate & KEEP_RECV_PAUSE) && data->state.tempwrite) {
-    /* we have a buffer for sending that we now seem to be able to deliver
-       since the receive pausing is lifted! */
-
-    /* get the pointer in local copy since the function may return PAUSE
-       again and then we'll get a new copy allocted and stored in
-       the tempwrite variables */
-    char *tempwrite = data->state.tempwrite;
-
-    data->state.tempwrite = NULL;
-    result = Curl_client_chop_write(data->easy_conn, data->state.tempwritetype,
-                                    tempwrite, data->state.tempwritesize);
-    free(tempwrite);
+  if(!(newstate & KEEP_RECV_PAUSE) && data->state.tempcount) {
+    /* there are buffers for sending that can be delivered as the receive
+       pausing is lifted! */
+    unsigned int i;
+    unsigned int count = data->state.tempcount;
+    struct tempbuf writebuf[3]; /* there can only be three */
+
+    /* copy the structs to allow for immediate re-pausing */
+    for(i=0; i < data->state.tempcount; i++) {
+      writebuf[i] = data->state.tempwrite[i];
+      data->state.tempwrite[i].buf = NULL;
+    }
+    data->state.tempcount = 0;
+
+    for(i=0; i < count; i++) {
+      /* even if one function returns error, this loops through and frees all
+         buffers */
+      if(!result)
+        result = Curl_client_chop_write(data->easy_conn,
+                                        writebuf[i].type,
+                                        writebuf[i].buf,
+                                        writebuf[i].len);
+      free(writebuf[i].buf);
+    }
+    if(result)
+      return result;
   }
 
   /* if there's no error and we're not pausing both directions, we want
diff --git a/lib/multi.c b/lib/multi.c
index 88ce00566..47e24005a 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -532,6 +532,7 @@ static CURLcode multi_done(struct connectdata **connp,
   CURLcode result;
   struct connectdata *conn;
   struct Curl_easy *data;
+  unsigned int i;
 
   DEBUGASSERT(*connp);
 
@@ -598,9 +599,11 @@ static CURLcode multi_done(struct connectdata **connp,
   }
 
   /* if the transfer was completed in a paused state there can be buffered
-     data left to write and then kill */
-  free(data->state.tempwrite);
-  data->state.tempwrite = NULL;
+     data left to free */
+  for(i=0; i < data->state.tempcount; i++) {
+    free(data->state.tempwrite[i].buf);
+  }
+  data->state.tempcount = 0;
 
   /* if data->set.reuse_forbid is TRUE, it means the libcurl client has
      forced us to close this connection. This is ignored for requests taking
diff --git a/lib/sendf.c b/lib/sendf.c
index e03241f0d..84b6b4b2f 100644
--- a/lib/sendf.c
+++ b/lib/sendf.c
@@ -33,6 +33,7 @@
 #include "non-ascii.h"
 #include "strerror.h"
 #include "select.h"
+#include "strdup.h"
 
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
@@ -474,21 +475,58 @@ static CURLcode pausewrite(struct Curl_easy *data,
      we want to send we need to dup it to save a copy for when the sending
      is again enabled */
   struct SingleRequest *k = &data->req;
-  char *dupl = malloc(len);
-  if(!dupl)
-    return CURLE_OUT_OF_MEMORY;
+  struct UrlState *s = &data->state;
+  char *dupl;
+  unsigned int i;
+  bool newtype = TRUE;
+
+  if(s->tempcount) {
+    for(i=0; i< s->tempcount; i++) {
+      if(s->tempwrite[i].type == type) {
+        /* data for this type exists */
+        newtype = FALSE;
+        break;
+      }
+    }
+    DEBUGASSERT(i < 3);
+  }
+  else
+    i = 0;
+
+  if(!newtype) {
+    /* append new data to old data */
+
+    /* figure out the new size of the data to save */
+    size_t newlen = len + s->tempwrite[i].len;
+    /* allocate the new memory area */
+    char *newptr = realloc(s->tempwrite[i].buf, newlen);
+    if(!newptr)
+      return CURLE_OUT_OF_MEMORY;
+    /* copy the new data to the end of the new area */
+    memcpy(newptr + s->tempwrite[i].len, ptr, len);
+
+    /* update the pointer and the size */
+    s->tempwrite[i].buf = newptr;
+    s->tempwrite[i].len = newlen;
+  }
+  else {
+    dupl = Curl_memdup(ptr, len);
+    if(!dupl)
+      return CURLE_OUT_OF_MEMORY;
 
-  memcpy(dupl, ptr, len);
+    /* store this information in the state struct for later use */
+    s->tempwrite[i].buf = dupl;
+    s->tempwrite[i].len = len;
+    s->tempwrite[i].type = type;
 
-  /* store this information in the state struct for later use */
-  data->state.tempwrite = dupl;
-  data->state.tempwritesize = len;
-  data->state.tempwritetype = type;
+    if(newtype)
+      s->tempcount++;
+  }
 
   /* mark the connection as RECV paused */
   k->keepon |= KEEP_RECV_PAUSE;
 
-  DEBUGF(infof(data, "Pausing with %zu bytes in buffer for type %02x\n",
+  DEBUGF(infof(data, "Paused %zu bytes in buffer for type %02x\n",
                len, type));
 
   return CURLE_OK;
@@ -511,31 +549,10 @@ CURLcode Curl_client_chop_write(struct connectdata *conn,
   if(!len)
     return CURLE_OK;
 
-  /* If reading is actually paused, we're forced to append this chunk of data
-     to the already held data, but only if it is the same type as otherwise it
-     can't work and it'll return error instead. */
-  if(data->req.keepon & KEEP_RECV_PAUSE) {
-    size_t newlen;
-    char *newptr;
-    if(type != data->state.tempwritetype)
-      /* major internal confusion */
-      return CURLE_RECV_ERROR;
-
-    DEBUGASSERT(data->state.tempwrite);
-
-    /* figure out the new size of the data to save */
-    newlen = len + data->state.tempwritesize;
-    /* allocate the new memory area */
-    newptr = realloc(data->state.tempwrite, newlen);
-    if(!newptr)
-      return CURLE_OUT_OF_MEMORY;
-    /* copy the new data to the end of the new area */
-    memcpy(newptr + data->state.tempwritesize, ptr, len);
-    /* update the pointer and the size */
-    data->state.tempwrite = newptr;
-    data->state.tempwritesize = newlen;
-    return CURLE_OK;
-  }
+  /* If reading is paused, append this data to the already held data for this
+     type. */
+  if(data->req.keepon & KEEP_RECV_PAUSE)
+    return pausewrite(data, type, ptr, len);
 
   /* Determine the callback(s) to use. */
   if(type & CLIENTWRITE_BODY)
@@ -615,6 +632,8 @@ CURLcode Curl_client_write(struct connectdata *conn,
   if(0 == len)
     len = strlen(ptr);
 
+  DEBUGASSERT(type <= 3);
+
   /* FTP data may need conversion. */
   if((type & CLIENTWRITE_BODY) &&
     (conn->handler->protocol & PROTO_FAMILY_FTP) &&
diff --git a/lib/urldata.h b/lib/urldata.h
index 3ec7e0f16..9b3849117 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1294,6 +1294,19 @@ struct Curl_http2_dep {
   struct Curl_easy *data;
 };
 
+/*
+ * This struct is for holding data that was attemped to get sent to the user's
+ * callback but is held due to pausing. One instance per type (BOTH, HEADER,
+ * BODY).
+ */
+struct tempbuf {
+  char *buf;  /* allocated buffer to keep data in when a write callback
+                 returns to make the connection paused */
+  size_t len; /* size of the 'tempwrite' allocated buffer */
+  int type;   /* type of the 'tempwrite' buffer as a bitmask that is used with
+                 Curl_client_write() */
+};
+
 struct UrlState {
 
   /* Points to the connection cache */
@@ -1327,11 +1340,8 @@ struct UrlState {
   int first_remote_port; /* remote port of the first (not followed) request */
   struct curl_ssl_session *session; /* array of 'max_ssl_sessions' size */
   long sessionage;                  /* number of the most recent session */
-  char *tempwrite;      /* allocated buffer to keep data in when a write
-                           callback returns to make the connection paused */
-  size_t tempwritesize; /* size of the 'tempwrite' allocated buffer */
-  int tempwritetype;    /* type of the 'tempwrite' buffer as a bitmask that is
-                           used with Curl_client_write() */
+  unsigned int tempcount; /* number of entries in use in tempwrite, 0 - 3 */
+  struct tempbuf tempwrite[3]; /* BOTH, HEADER, BODY */
   char *scratch; /* huge buffer[BUFSIZE*2] when doing upload CRLF replacing */
   bool errorbuf; /* Set to TRUE if the error buffer is already filled in.
                     This must be set to FALSE every time _easy_perform() is
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index edeb9000c..e908c12b9 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -163,6 +163,7 @@ test1520 \
 \
 test1525 test1526 test1527 test1528 test1529 test1530 test1531 test1532 \
 test1533 test1534 test1535 test1536 \
+test1540 \
 \
 test1600 test1601 test1602 test1603 test1604 test1605 \
 \
diff --git a/tests/data/test1540 b/tests/data/test1540
new file mode 100644
index 000000000..3277f550f
--- /dev/null
+++ b/tests/data/test1540
@@ -0,0 +1,64 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+CURLPAUSE_RECV
+chunked encoding
+Trailer:
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK swsclose

+Transfer-Encoding: chunked

+Trailer: MyCoolTrailerHeader

+

+4

+data

+5

+d474
+

+0

+MyCoolTrailerHeader: amazingtrailer

+

+</data>
+<datacheck>
+HTTP/1.1 200 OK swsclose

+Transfer-Encoding: chunked

+Trailer: MyCoolTrailerHeader

+

+Got 4 bytes but pausing!
+datad474
+MyCoolTrailerHeader: amazingtrailer

+</datacheck>
+
+</reply>
+# Client-side
+<client>
+<server>
+http
+</server>
+<tool>
+lib1540
+</tool>
+ <name>
+chunked with trailers and pausing the receive
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/1540
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /1540 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc
index 66628aae1..d85956d8b 100644
--- a/tests/libtest/Makefile.inc
+++ b/tests/libtest/Makefile.inc
@@ -25,6 +25,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib1520 \
  lib1525 lib1526 lib1527 lib1528 lib1529 lib1530 lib1531 lib1532 lib1533 \
  lib1534 lib1535 lib1536 \
+ lib1540 \
  lib1900 \
  lib2033
 
@@ -412,6 +413,10 @@ lib1536_SOURCES = lib1536.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1536_LDADD = $(TESTUTIL_LIBS)
 lib1536_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1536
 
+lib1540_SOURCES = lib1540.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib1540_LDADD = $(TESTUTIL_LIBS)
+lib1540_CPPFLAGS = $(AM_CPPFLAGS)
+
 lib1900_SOURCES = lib1900.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1900_LDADD = $(TESTUTIL_LIBS)
 lib1900_CPPFLAGS = $(AM_CPPFLAGS)
diff --git a/tests/libtest/lib1540.c b/tests/libtest/lib1540.c
new file mode 100644
index 000000000..86ba085ca
--- /dev/null
+++ b/tests/libtest/lib1540.c
@@ -0,0 +1,121 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "test.h"
+
+#include "testutil.h"
+#include "warnless.h"
+#include "memdebug.h"
+
+struct transfer_status {
+  CURL *easy;
+  int halted;
+  int counter; /* count write callback invokes */
+  int please;  /* number of times xferinfo is called while halted */
+};
+
+static int please_continue(void *userp,
+                           curl_off_t dltotal,
+                           curl_off_t dlnow,
+                           curl_off_t ultotal,
+                           curl_off_t ulnow)
+{
+  struct transfer_status *st = (struct transfer_status *)userp;
+  (void)dltotal;
+  (void)dlnow;
+  (void)ultotal;
+  (void)ulnow;
+  if(st->halted) {
+    st->please++;
+    if(st->please == 2) {
+      /* waited enough, unpause! */
+      curl_easy_pause(st->easy, CURLPAUSE_CONT);
+    }
+  }
+  fprintf(stderr, "xferinfo: paused %d\n", st->halted);
+  return 0; /* go on */
+}
+
+static size_t header_callback(void *ptr, size_t size, size_t nmemb,
+                              void *userp)
+{
+  size_t len = size * nmemb;
+  (void)userp;
+  (void)fwrite(ptr, size, nmemb, stdout);
+  return len;
+}
+
+static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp)
+{
+  struct transfer_status *st = (struct transfer_status *)userp;
+  size_t len = size * nmemb;
+  st->counter++;
+  if(st->counter > 1) {
+    /* the first call puts us on pause, so subsequent calls are after
+       unpause */
+    fwrite(ptr, size, nmemb, stdout);
+    return len;
+  }
+  printf("Got %d bytes but pausing!\n", (int)len);
+  st->halted = 1;
+  return CURL_WRITEFUNC_PAUSE;
+}
+
+#define TEST_HANG_TIMEOUT 60 * 1000
+
+int test(char *URL)
+{
+  CURL *curls = NULL;
+  int i = 0;
+  int res = 0;
+  struct transfer_status st;
+
+  start_test_timing();
+
+  memset(&st, 0, sizeof(st));
+
+  global_init(CURL_GLOBAL_ALL);
+
+  easy_init(curls);
+  st.easy = curls; /* to allow callbacks access */
+
+  easy_setopt(curls, CURLOPT_URL, URL);
+  easy_setopt(curls, CURLOPT_WRITEFUNCTION, write_callback);
+  easy_setopt(curls, CURLOPT_WRITEDATA, &st);
+  easy_setopt(curls, CURLOPT_HEADERFUNCTION, header_callback);
+  easy_setopt(curls, CURLOPT_HEADERDATA, &st);
+
+  easy_setopt(curls, CURLOPT_XFERINFOFUNCTION, please_continue);
+  easy_setopt(curls, CURLOPT_XFERINFODATA, &st);
+  easy_setopt(curls, CURLOPT_NOPROGRESS, 0L);
+
+  res = curl_easy_perform(curls);
+
+test_cleanup:
+
+  curl_easy_cleanup(curls);
+  curl_global_cleanup();
+
+  if(res)
+    i = res;
+
+  return i; /* return the final return code */
+}