url: alloc the download buffer at transfer start

... and free it as soon as the transfer is done. It removes the extra
alloc when a new size is set with setopt() and reduces memory for unused
easy handles.

In addition: the closure_handle now doesn't use an allocated buffer at
all but the smallest supported size as a stack based one.

Closes #5472

8 files changed, 50 insertions, 53 deletions

diff --git a/lib/conncache.c b/lib/conncache.c
index 95fcea6f3..4474c28b4 100644
--- a/lib/conncache.c
+++ b/lib/conncache.c
@@ -531,6 +531,11 @@ Curl_conncache_extract_oldest(struct Curl_easy *data)
 void Curl_conncache_close_all_connections(struct conncache *connc)
 {
   struct connectdata *conn;
+  char buffer[READBUFFER_MIN];
+  if(!connc->closure_handle)
+    return;
+  connc->closure_handle->state.buffer = buffer;
+  connc->closure_handle->set.buffer_size = READBUFFER_MIN;
 
   conn = conncache_find_first_connection(connc);
   while(conn) {
@@ -547,6 +552,7 @@ void Curl_conncache_close_all_connections(struct conncache *connc)
     conn = conncache_find_first_connection(connc);
   }
 
+  connc->closure_handle->state.buffer = NULL;
   if(connc->closure_handle) {
     SIGPIPE_VARIABLE(pipe_st);
     sigpipe_ignore(connc->closure_handle, &pipe_st);
diff --git a/lib/easy.c b/lib/easy.c
index 1deedb265..f58c4521a 100644
--- a/lib/easy.c
+++ b/lib/easy.c
@@ -829,9 +829,6 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
    * the likeliness of us forgetting to init a buffer here in the future.
    */
   outcurl->set.buffer_size = data->set.buffer_size;
-  outcurl->state.buffer = malloc(outcurl->set.buffer_size + 1);
-  if(!outcurl->state.buffer)
-    goto fail;
 
   /* copy all userdefined values */
   if(dupset(outcurl, data))
@@ -947,8 +944,6 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
  */
 void curl_easy_reset(struct Curl_easy *data)
 {
-  long old_buffer_size = data->set.buffer_size;
-
   Curl_free_request_state(data);
 
   /* zero out UserDefined data: */
@@ -972,18 +967,6 @@ void curl_easy_reset(struct Curl_easy *data)
 #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH)
   Curl_http_auth_cleanup_digest(data);
 #endif
-
-  /* resize receive buffer */
-  if(old_buffer_size != data->set.buffer_size) {
-    char *newbuff = realloc(data->state.buffer, data->set.buffer_size + 1);
-    if(!newbuff) {
-      DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n"));
-      /* nothing we can do here except use the old size */
-      data->set.buffer_size = old_buffer_size;
-    }
-    else
-      data->state.buffer = newbuff;
-  }
 }
 
 /*
diff --git a/lib/http2.c b/lib/http2.c
index e4733c94b..7d56e2280 100644
--- a/lib/http2.c
+++ b/lib/http2.c
@@ -258,15 +258,14 @@ static unsigned int http2_conncheck(struct connectdata *check,
 void Curl_http2_setup_req(struct Curl_easy *data)
 {
   struct HTTP *http = data->req.protop;
-
   http->bodystarted = FALSE;
   http->status_code = -1;
   http->pausedata = NULL;
   http->pauselen = 0;
   http->closed = FALSE;
   http->close_handled = FALSE;
-  http->mem = data->state.buffer;
-  http->len = data->set.buffer_size;
+  http->mem = NULL;
+  http->len = 0;
   http->memlen = 0;
 }
 
@@ -2103,6 +2102,8 @@ CURLcode Curl_http2_setup(struct connectdata *conn)
   struct http_conn *httpc = &conn->proto.httpc;
   struct HTTP *stream = conn->data->req.protop;
 
+  DEBUGASSERT(conn->data->state.buffer);
+
   stream->stream_id = -1;
 
   Curl_dyn_init(&stream->header_recvbuf, DYN_H2_HEADERS);
@@ -2126,6 +2127,8 @@ CURLcode Curl_http2_setup(struct connectdata *conn)
   stream->upload_left = 0;
   stream->upload_mem = NULL;
   stream->upload_len = 0;
+  stream->mem = conn->data->state.buffer;
+  stream->len = conn->data->set.buffer_size;
 
   httpc->inbuflen = 0;
   httpc->nread_inbuf = 0;
diff --git a/lib/multi.c b/lib/multi.c
index 75b0357c3..b753b1a61 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -678,6 +678,7 @@ static CURLcode multi_done(struct Curl_easy *data,
       data->state.lastconnect = NULL;
   }
 
+  Curl_safefree(data->state.buffer);
   Curl_free_request_state(data);
   return result;
 }
@@ -1522,6 +1523,20 @@ static CURLcode protocol_connect(struct connectdata *conn,
   return result; /* pass back status */
 }
 
+/*
+ * preconnect() is called immediately before a connect starts. When a redirect
+ * is followed, this is then called multiple times during a single transfer.
+ */
+static CURLcode preconnect(struct Curl_easy *data)
+{
+  if(!data->state.buffer) {
+    data->state.buffer = malloc(data->set.buffer_size + 1);
+    if(!data->state.buffer)
+      return CURLE_OUT_OF_MEMORY;
+  }
+  return CURLE_OK;
+}
+
 
 static CURLMcode multi_runsingle(struct Curl_multi *multi,
                                  struct curltime now,
@@ -1629,6 +1644,11 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
 
     case CURLM_STATE_CONNECT:
       /* Connect. We want to get a connection identifier filled in. */
+      /* init this transfer. */
+      result = preconnect(data);
+      if(result)
+        break;
+
       Curl_pgrsTime(data, TIMER_STARTSINGLE);
       if(data->set.timeout)
         Curl_expire(data, data->set.timeout, EXPIRE_TIMEOUT);
@@ -2058,7 +2078,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       char *newurl = NULL;
       bool retry = FALSE;
       bool comeback = FALSE;
-
+      DEBUGASSERT(data->state.buffer);
       /* check if over send speed */
       send_timeout_ms = 0;
       if(data->set.max_send_speed > 0)
diff --git a/lib/setopt.c b/lib/setopt.c
index 72704127c..2ddaeef7d 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -2076,7 +2076,7 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
       arg = READBUFFER_MIN;
 
     /* Resize if new size */
-    if(arg != data->set.buffer_size) {
+    if((arg != data->set.buffer_size) && data->state.buffer) {
       char *newbuff = realloc(data->state.buffer, arg + 1);
       if(!newbuff) {
         DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n"));
diff --git a/lib/transfer.c b/lib/transfer.c
index dc43cf6ce..ea337ea1d 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -558,6 +558,8 @@ static CURLcode readwrite_data(struct Curl_easy *data,
   size_t excess = 0; /* excess bytes read */
   bool readmore = FALSE; /* used by RTP to signal for more data */
   int maxloops = 100;
+  char *buf = data->state.buffer;
+  DEBUGASSERT(buf);
 
   *done = FALSE;
   *comeback = FALSE;
@@ -592,7 +594,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
 
     if(bytestoread) {
       /* receive data from the network! */
-      result = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread);
+      result = Curl_read(conn, conn->sockfd, buf, bytestoread, &nread);
 
       /* read would've blocked */
       if(CURLE_AGAIN == result)
@@ -619,9 +621,8 @@ static CURLcode readwrite_data(struct Curl_easy *data,
     /* indicates data of zero size, i.e. empty file */
     is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE;
 
-    /* NUL terminate, allowing string ops to be used */
     if(0 < nread || is_empty_data) {
-      k->buf[nread] = 0;
+      buf[nread] = 0;
     }
     else {
       /* if we receive 0 or less here, either the http2 stream is closed or the
@@ -638,7 +639,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
 
     /* Default buffer to use when we write the buffer, it may be changed
        in the flow below before the actual storing is done. */
-    k->str = k->buf;
+    k->str = buf;
 
     if(conn->handler->readwrite) {
       result = conn->handler->readwrite(data, conn, &nread, &readmore);
@@ -905,10 +906,10 @@ static CURLcode readwrite_data(struct Curl_easy *data,
       /* Parse the excess data */
       k->str += nread;
 
-      if(&k->str[excess] > &k->buf[data->set.buffer_size]) {
+      if(&k->str[excess] > &buf[data->set.buffer_size]) {
         /* the excess amount was too excessive(!), make sure
            it doesn't read out of buffer */
-        excess = &k->buf[data->set.buffer_size] - k->str;
+        excess = &buf[data->set.buffer_size] - k->str;
       }
       nread = (ssize_t)excess;
 
@@ -1467,7 +1468,6 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
   data->state.authhost.want = data->set.httpauth;
   data->state.authproxy.want = data->set.proxyauth;
   Curl_safefree(data->info.wouldredirect);
-  data->info.wouldredirect = NULL;
 
   if(data->set.httpreq == HTTPREQ_PUT)
     data->state.infilesize = data->set.filesize;
diff --git a/lib/url.c b/lib/url.c
index 5114deaea..578844bae 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -610,31 +610,21 @@ CURLcode Curl_open(struct Curl_easy **curl)
     return result;
   }
 
-  /* We do some initial setup here, all those fields that can't be just 0 */
-
-  data->state.buffer = malloc(READBUFFER_SIZE + 1);
-  if(!data->state.buffer) {
-    DEBUGF(fprintf(stderr, "Error: malloc of buffer failed\n"));
-    result = CURLE_OUT_OF_MEMORY;
-  }
-  else {
-    result = Curl_init_userdefined(data);
-    if(!result) {
-      Curl_dyn_init(&data->state.headerb, CURL_MAX_HTTP_HEADER);
-      Curl_convert_init(data);
-      Curl_initinfo(data);
+  result = Curl_init_userdefined(data);
+  if(!result) {
+    Curl_dyn_init(&data->state.headerb, CURL_MAX_HTTP_HEADER);
+    Curl_convert_init(data);
+    Curl_initinfo(data);
 
-      /* most recent connection is not yet defined */
-      data->state.lastconnect = NULL;
+    /* most recent connection is not yet defined */
+    data->state.lastconnect = NULL;
 
-      data->progress.flags |= PGRS_HIDE;
-      data->state.current_speed = -1; /* init to negative == impossible */
-    }
+    data->progress.flags |= PGRS_HIDE;
+    data->state.current_speed = -1; /* init to negative == impossible */
   }
 
   if(result) {
     Curl_resolver_cleanup(data->state.resolver);
-    free(data->state.buffer);
     Curl_dyn_free(&data->state.headerb);
     Curl_freeset(data);
     free(data);
@@ -3973,14 +3963,10 @@ CURLcode Curl_init_do(struct Curl_easy *data, struct connectdata *conn)
   k->start = Curl_now(); /* start time */
   k->now = k->start;   /* current time is now */
   k->header = TRUE; /* assume header */
-
   k->bytecount = 0;
-
-  k->buf = data->state.buffer;
   k->ignorebody = FALSE;
 
   Curl_speedinit(data);
-
   Curl_pgrsSetUploadCounter(data, 0);
   Curl_pgrsSetDownloadCounter(data, 0);
 
diff --git a/lib/urldata.h b/lib/urldata.h
index 20ce5ed10..0d756f18e 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -627,7 +627,6 @@ struct SingleRequest {
   struct contenc_writer *writer_stack;
   time_t timeofdoc;
   long bodywrites;
-  char *buf;
   int keepon;
   char *location;   /* This points to an allocated version of the Location:
                        header data */