http-proxy: do the HTTP CONNECT process entirely non-blocking

Mentioned as a problem since 2007 (8f87c15bdac63) and of course it
existed even before that.

Closes #1547

9 files changed, 177 insertions, 133 deletions

diff --git a/docs/libcurl/libcurl-multi.3 b/docs/libcurl/libcurl-multi.3
index d25a1b00c..76f8c3620 100644
--- a/docs/libcurl/libcurl-multi.3
+++ b/docs/libcurl/libcurl-multi.3
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -173,7 +173,6 @@ the future, you should be aware of the following current restrictions:
 
 .nf
  - Name resolves unless the c-ares or threaded-resolver backends are used
- - HTTP proxy CONNECT operations
  - SOCKS proxy handshakes
  - file:// transfers
  - TELNET transfers
diff --git a/lib/ftp.c b/lib/ftp.c
index 5edec3761..f929c3407 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -272,7 +272,6 @@ static void close_secondarysocket(struct connectdata *conn)
     conn->sock[SECONDARYSOCKET] = CURL_SOCKET_BAD;
   }
   conn->bits.tcpconnect[SECONDARYSOCKET] = FALSE;
-  conn->tunnel_state[SECONDARYSOCKET] = TUNNEL_INIT;
 }
 
 /*
@@ -3585,7 +3584,7 @@ static CURLcode ftp_do_more(struct connectdata *conn, int *completep)
 
   /* if the second connection isn't done yet, wait for it */
   if(!conn->bits.tcpconnect[SECONDARYSOCKET]) {
-    if(conn->tunnel_state[SECONDARYSOCKET] == TUNNEL_CONNECT) {
+    if(Curl_connect_ongoing(conn)) {
       /* As we're in TUNNEL_CONNECT state now, we know the proxy name and port
          aren't used so we blank their arguments. TODO: make this nicer */
       result = Curl_proxyCONNECT(conn, SECONDARYSOCKET, NULL, 0);
@@ -3617,7 +3616,7 @@ static CURLcode ftp_do_more(struct connectdata *conn, int *completep)
     return result;
 
   if(conn->bits.tunnel_proxy && conn->bits.httpproxy &&
-     conn->tunnel_state[SECONDARYSOCKET] != TUNNEL_COMPLETE)
+     Curl_connect_ongoing(conn))
     return result;
 
 
diff --git a/lib/http.c b/lib/http.c
index 21574e21d..c65cb58d2 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -1369,7 +1369,7 @@ CURLcode Curl_http_connect(struct connectdata *conn, bool *done)
   if(CONNECT_FIRSTSOCKET_PROXY_SSL())
     return CURLE_OK; /* wait for HTTPS proxy SSL initialization to complete */
 
-  if(conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)
+  if(!Curl_connect_complete(conn))
     /* nothing else to do except wait right now - we're not done here. */
     return CURLE_OK;
 
diff --git a/lib/http_proxy.c b/lib/http_proxy.c
index 9894e2e5f..cd5e7c269 100644
--- a/lib/http_proxy.c
+++ b/lib/http_proxy.c
@@ -135,7 +135,47 @@ CURLcode Curl_proxy_connect(struct connectdata *conn, int sockindex)
   return CURLE_OK;
 }
 
-#define CONNECT_BUFFER_SIZE 16384
+bool Curl_connect_complete(struct connectdata *conn)
+{
+  return conn->connect_state &&
+    (conn->connect_state->tunnel_state == TUNNEL_COMPLETE);
+}
+
+bool Curl_connect_ongoing(struct connectdata *conn)
+{
+  return conn->connect_state &&
+    (conn->connect_state->tunnel_state != TUNNEL_COMPLETE);
+}
+
+static CURLcode connect_init(struct connectdata *conn, bool reinit)
+{
+  struct http_connect_state *s;
+  if(!reinit) {
+    DEBUGASSERT(!conn->connect_state);
+    s = calloc(1, sizeof(struct http_connect_state));
+    if(!s)
+      return CURLE_OUT_OF_MEMORY;
+    infof(conn->data, "allocate connect buffer!\n");
+    conn->connect_state = s;
+  }
+  else {
+    DEBUGASSERT(conn->connect_state);
+    s = conn->connect_state;
+  }
+  s->tunnel_state = TUNNEL_INIT;
+  s->keepon=TRUE;
+  s->line_start = s->connect_buffer;
+  s->ptr = s->line_start;
+  s->cl=0;
+  return CURLE_OK;
+}
+
+static void connect_done(struct connectdata *conn)
+{
+  struct http_connect_state *s = conn->connect_state;
+  s->tunnel_state = TUNNEL_COMPLETE;
+  infof(conn->data, "CONNECT phase completed!\n");
+}
 
 static CURLcode CONNECT(struct connectdata *conn,
                         int sockindex,
@@ -147,23 +187,22 @@ static CURLcode CONNECT(struct connectdata *conn,
   struct SingleRequest *k = &data->req;
   CURLcode result;
   curl_socket_t tunnelsocket = conn->sock[sockindex];
-  curl_off_t cl=0;
   bool closeConnection = FALSE;
   bool chunked_encoding = FALSE;
   time_t check;
+  struct http_connect_state *s = conn->connect_state;
 
 #define SELECT_OK      0
 #define SELECT_ERROR   1
 #define SELECT_TIMEOUT 2
-  int error = SELECT_OK;
 
-  if(conn->tunnel_state[sockindex] == TUNNEL_COMPLETE)
+  if(Curl_connect_complete(conn))
     return CURLE_OK; /* CONNECT is already completed */
 
   conn->bits.proxy_connect_closed = FALSE;
 
   do {
-    if(TUNNEL_INIT == conn->tunnel_state[sockindex]) {
+    if(TUNNEL_INIT == s->tunnel_state) {
       /* BEGIN CONNECT PHASE */
       char *host_port;
       Curl_send_buffer *req_buffer;
@@ -271,65 +310,47 @@ static CURLcode CONNECT(struct connectdata *conn,
       if(result)
         return result;
 
-      conn->tunnel_state[sockindex] = TUNNEL_CONNECT;
+      s->tunnel_state = TUNNEL_CONNECT;
     } /* END CONNECT PHASE */
 
     check = Curl_timeleft(data, NULL, TRUE);
     if(check <= 0) {
       failf(data, "Proxy CONNECT aborted due to timeout");
-      return CURLE_RECV_ERROR;
+      return CURLE_OPERATION_TIMEDOUT;
     }
 
     if(!Curl_conn_data_pending(conn, sockindex))
       /* return so we'll be called again polling-style */
       return CURLE_OK;
-    DEBUGF(infof(data, "Read response immediately from proxy CONNECT\n"));
 
     /* at this point, the tunnel_connecting phase is over. */
 
     { /* READING RESPONSE PHASE */
-      size_t nread;   /* total size read */
-      int perline; /* count bytes per line */
-      int keepon=TRUE;
-      ssize_t gotbytes;
-      char *ptr;
-      char *line_start;
+      int error = SELECT_OK;
 
-      ptr = conn->connect_buffer;
-      line_start = ptr;
+      s->perline = 0;
 
-      nread = 0;
-      perline = 0;
+      while(s->keepon && !error) {
+        ssize_t gotbytes;
 
-      while(nread < (size_t)CONNECT_BUFFER_SIZE && keepon && !error) {
-        if(Curl_pgrsUpdate(conn))
-          return CURLE_ABORTED_BY_CALLBACK;
-
-        if(ptr >= &conn->connect_buffer[CONNECT_BUFFER_SIZE]) {
+        /* make sure we have space to read more data */
+        if(s->ptr >= &s->connect_buffer[CONNECT_BUFFER_SIZE]) {
           failf(data, "CONNECT response too large!");
           return CURLE_RECV_ERROR;
         }
 
-        check = Curl_timeleft(data, NULL, TRUE);
-        if(check <= 0) {
-          failf(data, "Proxy CONNECT aborted due to timeout");
-          error = SELECT_TIMEOUT; /* already too little time */
-          break;
-        }
-
         /* Read one byte at a time to avoid a race condition. Wait at most one
            second before looping to ensure continuous pgrsUpdates. */
-        result = Curl_read(conn, tunnelsocket, ptr, 1, &gotbytes);
-        if(result == CURLE_AGAIN) {
-          if(SOCKET_READABLE(tunnelsocket, check<1000L?check:1000) == -1) {
-            error = SELECT_ERROR;
-            failf(data, "Proxy CONNECT aborted due to select/poll error");
-            break;
-          }
-          continue;
-        }
+        result = Curl_read(conn, tunnelsocket, s->ptr, 1, &gotbytes);
+        if(result == CURLE_AGAIN)
+          /* socket buffer drained, return */
+          return CURLE_OK;
+
+        if(Curl_pgrsUpdate(conn))
+          return CURLE_ABORTED_BY_CALLBACK;
+
         if(result) {
-          keepon = FALSE;
+          s->keepon = FALSE;
           break;
         }
         else if(gotbytes <= 0) {
@@ -343,24 +364,22 @@ static CURLcode CONNECT(struct connectdata *conn,
             error = SELECT_ERROR;
             failf(data, "Proxy CONNECT aborted");
           }
-          keepon = FALSE;
+          s->keepon = FALSE;
           break;
         }
 
-        /* We got a byte of data */
-        nread++;
 
-        if(keepon > TRUE) {
+        if(s->keepon > TRUE) {
           /* This means we are currently ignoring a response-body */
 
-          nread = 0; /* make next read start over in the read buffer */
-          ptr = conn->connect_buffer;
-          if(cl) {
+          s->ptr = s->connect_buffer;
+          if(s->cl) {
             /* A Content-Length based body: simply count down the counter
                and make sure to break out of the loop when we're done! */
-            cl--;
-            if(cl <= 0) {
-              keepon = FALSE;
+            s->cl--;
+            if(s->cl <= 0) {
+              s->keepon = FALSE;
+              s->tunnel_state = TUNNEL_COMPLETE;
               break;
             }
           }
@@ -372,23 +391,23 @@ static CURLcode CONNECT(struct connectdata *conn,
 
             /* now parse the chunked piece of data so that we can
                properly tell when the stream ends */
-            r = Curl_httpchunk_read(conn, ptr, 1, &tookcareof);
+            r = Curl_httpchunk_read(conn, s->ptr, 1, &tookcareof);
             if(r == CHUNKE_STOP) {
               /* we're done reading chunks! */
               infof(data, "chunk reading DONE\n");
-              keepon = FALSE;
+              s->keepon = FALSE;
               /* we did the full CONNECT treatment, go COMPLETE */
-              conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
+              s->tunnel_state = TUNNEL_COMPLETE;
             }
           }
           continue;
         }
 
-        perline++; /* amount of bytes in this line so far */
+        s->perline++; /* amount of bytes in this line so far */
 
         /* if this is not the end of a header line then continue */
-        if(*ptr != 0x0a) {
-          ptr++;
+        if(*s->ptr != 0x0a) {
+          s->ptr++;
           continue;
         }
 
@@ -401,7 +420,7 @@ static CURLcode CONNECT(struct connectdata *conn,
         /* output debug if that is requested */
         if(data->set.verbose)
           Curl_debug(data, CURLINFO_HEADER_IN,
-                     line_start, (size_t)perline, conn);
+                     s->line_start, (size_t)s->perline, conn);
 
         if(!data->set.suppress_connect_headers) {
           /* send the header to the callback */
@@ -409,33 +428,32 @@ static CURLcode CONNECT(struct connectdata *conn,
           if(data->set.include_header)
             writetype |= CLIENTWRITE_BODY;
 
-          result = Curl_client_write(conn, writetype, line_start, perline);
+          result = Curl_client_write(conn, writetype,
+                                     s->line_start, s->perline);
           if(result)
             return result;
         }
 
-        data->info.header_size += (long)perline;
-        data->req.headerbytecount += (long)perline;
+        data->info.header_size += (long)s->perline;
+        data->req.headerbytecount += (long)s->perline;
 
         /* Newlines are CRLF, so the CR is ignored as the line isn't
            really terminated until the LF comes. Treat a following CR
            as end-of-headers as well.*/
 
-        if(('\r' == line_start[0]) ||
-           ('\n' == line_start[0])) {
+        if(('\r' == s->line_start[0]) ||
+           ('\n' == s->line_start[0])) {
           /* end of response-headers from the proxy */
-          nread = 0; /* make next read start over in the read
-                        buffer */
-          ptr = conn->connect_buffer;
+          s->ptr = s->connect_buffer;
           if((407 == k->httpcode) && !data->state.authproblem) {
             /* If we get a 407 response code with content length
                when we have no auth problem, we must ignore the
                whole response-body */
-            keepon = 2;
+            s->keepon = 2;
 
-            if(cl) {
+            if(s->cl) {
               infof(data, "Ignore %" CURL_FORMAT_CURL_OFF_T
-                    " bytes of response-body\n", cl);
+                    " bytes of response-body\n", s->cl);
             }
             else if(chunked_encoding) {
               CHUNKcode r;
@@ -448,46 +466,46 @@ static CURLcode CONNECT(struct connectdata *conn,
                  function returns! */
               k->ignorebody = TRUE;
 
-              if(line_start[1] == '\n') {
+              if(s->line_start[1] == '\n') {
                 /* this can only be a LF if the letter at index 0
                    was a CR */
-                line_start++;
+                s->line_start++;
               }
 
               /* now parse the chunked piece of data so that we can
                  properly tell when the stream ends */
-              r = Curl_httpchunk_read(conn, line_start + 1, 1, &gotbytes);
+              r = Curl_httpchunk_read(conn, s->line_start + 1, 1, &gotbytes);
               if(r == CHUNKE_STOP) {
                 /* we're done reading chunks! */
                 infof(data, "chunk reading DONE\n");
-                keepon = FALSE;
-                /* we did the full CONNECT treatment, go to
-                   COMPLETE */
-                conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
+                s->keepon = FALSE;
+                /* we did the full CONNECT treatment, go to COMPLETE */
+                s->tunnel_state = TUNNEL_COMPLETE;
               }
             }
             else {
               /* without content-length or chunked encoding, we
                  can't keep the connection alive since the close is
                  the end signal so we bail out at once instead */
-              keepon = FALSE;
+              s->keepon = FALSE;
             }
           }
           else
-            keepon = FALSE;
-          /* we did the full CONNECT treatment, go to COMPLETE */
-          conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
+            s->keepon = FALSE;
+          if(!s->cl)
+            /* we did the full CONNECT treatment, go to COMPLETE */
+            s->tunnel_state = TUNNEL_COMPLETE;
           continue;
         }
 
-        line_start[perline] = 0; /* zero terminate the buffer */
-        if((checkprefix("WWW-Authenticate:", line_start) &&
+        s->line_start[s->perline] = 0; /* zero terminate the buffer */
+        if((checkprefix("WWW-Authenticate:", s->line_start) &&
             (401 == k->httpcode)) ||
-           (checkprefix("Proxy-authenticate:", line_start) &&
+           (checkprefix("Proxy-authenticate:", s->line_start) &&
             (407 == k->httpcode))) {
 
           bool proxy = (k->httpcode == 407) ? TRUE : FALSE;
-          char *auth = Curl_copy_header_value(line_start);
+          char *auth = Curl_copy_header_value(s->line_start);
           if(!auth)
             return CURLE_OUT_OF_MEMORY;
 
@@ -498,7 +516,7 @@ static CURLcode CONNECT(struct connectdata *conn,
           if(result)
             return result;
         }
-        else if(checkprefix("Content-Length:", line_start)) {
+        else if(checkprefix("Content-Length:", s->line_start)) {
           if(k->httpcode/100 == 2) {
             /* A client MUST ignore any Content-Length or Transfer-Encoding
                header fields received in a successful response to CONNECT.
@@ -507,13 +525,13 @@ static CURLcode CONNECT(struct connectdata *conn,
                   k->httpcode);
           }
           else {
-            cl = curlx_strtoofft(line_start +
-                                 strlen("Content-Length:"), NULL, 10);
+            s->cl = curlx_strtoofft(s->line_start +
+                                    strlen("Content-Length:"), NULL, 10);
           }
         }
-        else if(Curl_compareheader(line_start, "Connection:", "close"))
+        else if(Curl_compareheader(s->line_start, "Connection:", "close"))
           closeConnection = TRUE;
-        else if(checkprefix("Transfer-Encoding:", line_start)) {
+        else if(checkprefix("Transfer-Encoding:", s->line_start)) {
           if(k->httpcode/100 == 2) {
             /* A client MUST ignore any Content-Length or Transfer-Encoding
                header fields received in a successful response to CONNECT.
@@ -521,7 +539,7 @@ static CURLcode CONNECT(struct connectdata *conn,
             infof(data, "Ignoring Transfer-Encoding in "
                   "CONNECT %03d response\n", k->httpcode);
           }
-          else if(Curl_compareheader(line_start,
+          else if(Curl_compareheader(s->line_start,
                                      "Transfer-Encoding:", "chunked")) {
             infof(data, "CONNECT responded chunked\n");
             chunked_encoding = TRUE;
@@ -529,18 +547,19 @@ static CURLcode CONNECT(struct connectdata *conn,
             Curl_httpchunk_init(conn);
           }
         }
-        else if(Curl_compareheader(line_start, "Proxy-Connection:", "close"))
+        else if(Curl_compareheader(s->line_start,
+                                   "Proxy-Connection:", "close"))
           closeConnection = TRUE;
-        else if(2 == sscanf(line_start, "HTTP/1.%d %d",
+        else if(2 == sscanf(s->line_start, "HTTP/1.%d %d",
                             &subversion,
                             &k->httpcode)) {
           /* store the HTTP code from the proxy */
           data->info.httpproxycode = k->httpcode;
         }
 
-        perline = 0; /* line starts over here */
-        ptr = conn->connect_buffer;
-        line_start = ptr;
+        s->perline = 0; /* line starts over here */
+        s->ptr = s->connect_buffer;
+        s->line_start = s->ptr;
       } /* while there's buffer left and loop is requested */
 
       if(Curl_pgrsUpdate(conn))
@@ -574,11 +593,8 @@ static CURLcode CONNECT(struct connectdata *conn,
     /* If we are supposed to continue and request a new URL, which basically
      * means the HTTP authentication is still going on so if the tunnel
      * is complete we start over in INIT state */
-    if(data->req.newurl &&
-       (TUNNEL_COMPLETE == conn->tunnel_state[sockindex])) {
-      conn->tunnel_state[sockindex] = TUNNEL_INIT;
-      infof(data, "TUNNEL_STATE switched to: %d\n",
-            conn->tunnel_state[sockindex]);
+    if(data->req.newurl && (TUNNEL_COMPLETE == s->tunnel_state)) {
+      connect_init(conn, TRUE); /* reinit */
     }
 
   } while(data->req.newurl);
@@ -587,6 +603,7 @@ static CURLcode CONNECT(struct connectdata *conn,
     if(closeConnection && data->req.newurl) {
       conn->bits.proxy_connect_closed = TRUE;
       infof(data, "Connect me again please\n");
+      connect_done(conn);
     }
     else {
       free(data->req.newurl);
@@ -598,7 +615,7 @@ static CURLcode CONNECT(struct connectdata *conn,
     }
 
     /* to back to init state */
-    conn->tunnel_state[sockindex] = TUNNEL_INIT;
+    s->tunnel_state = TUNNEL_INIT;
 
     if(conn->bits.proxy_connect_closed)
       /* this is not an error, just part of the connection negotiation */
@@ -608,7 +625,7 @@ static CURLcode CONNECT(struct connectdata *conn,
     return CURLE_RECV_ERROR;
   }
 
-  conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
+  s->tunnel_state = TUNNEL_COMPLETE;
 
   /* If a proxy-authorization header was used for the proxy, then we should
      make sure that it isn't accidentally used for the document request
@@ -625,6 +642,16 @@ static CURLcode CONNECT(struct connectdata *conn,
   return CURLE_OK;
 }
 
+void Curl_connect_free(struct Curl_easy *data)
+{
+  struct connectdata *conn = data->easy_conn;
+  struct http_connect_state *s = conn->connect_state;
+  if(s) {
+    free(s);
+    conn->connect_state = NULL;
+  }
+}
+
 /*
  * Curl_proxyCONNECT() requires that we're connected to a HTTP proxy. This
  * function will issue the necessary commands to get a seamless tunnel through
@@ -637,17 +664,15 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                            int remote_port)
 {
   CURLcode result;
-  if(TUNNEL_INIT == conn->tunnel_state[sockindex]) {
-    if(!conn->connect_buffer) {
-      conn->connect_buffer = malloc(CONNECT_BUFFER_SIZE);
-      if(!conn->connect_buffer)
-        return CURLE_OUT_OF_MEMORY;
-    }
+  if(!conn->connect_state) {
+    result = connect_init(conn, FALSE);
+    if(result)
+      return result;
   }
   result = CONNECT(conn, sockindex, hostname, remote_port);
 
-  if(result || (TUNNEL_COMPLETE == conn->tunnel_state[sockindex]))
-    Curl_safefree(conn->connect_buffer);
+  if(result || Curl_connect_complete(conn))
+    connect_done(conn);
 
   return result;
 }
diff --git a/lib/http_proxy.h b/lib/http_proxy.h
index cbb1ab421..bd9d6f590 100644
--- a/lib/http_proxy.h
+++ b/lib/http_proxy.h
@@ -33,6 +33,10 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
 
 CURLcode Curl_proxy_connect(struct connectdata *conn, int sockindex);
 
+bool Curl_connect_complete(struct connectdata *conn);
+bool Curl_connect_ongoing(struct connectdata *conn);
+void Curl_connect_free(struct Curl_easy *data);
+
 #else
 #define Curl_proxyCONNECT(x,y,z,w) CURLE_NOT_BUILT_IN
 #define Curl_proxy_connect(x,y) CURLE_OK
diff --git a/lib/multi.c b/lib/multi.c
index c3a0d122c..5753f58f7 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -44,6 +44,7 @@
 #include "sigpipe.h"
 #include "vtls/vtls.h"
 #include "connect.h"
+#include "http_proxy.h"
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
 #include "curl_memory.h"
@@ -114,6 +115,13 @@ static void mstate(struct Curl_easy *data, CURLMstate state
     NULL,
     NULL,
     Curl_init_CONNECT, /* CONNECT */
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    Curl_connect_free /* DO */
     /* the rest is NULL too */
   };
 
@@ -826,7 +834,7 @@ static int waitproxyconnect_getsock(struct connectdata *conn,
 
   /* when we've sent a CONNECT to a proxy, we should rather wait for the
      socket to become readable to be able to get the response headers */
-  if(conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)
+  if(conn->connect_state)
     return GETSOCK_READSOCK(0);
 
   return GETSOCK_WRITESOCK(0);
@@ -1455,7 +1463,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
                          CURLM_STATE_WAITDO:CURLM_STATE_DO);
             else {
 #ifndef CURL_DISABLE_HTTP
-              if(data->easy_conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)
+              if(Curl_connect_ongoing(data->easy_conn))
                 multistate(data, CURLM_STATE_WAITPROXYCONNECT);
               else
 #endif
@@ -1520,7 +1528,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
                        CURLM_STATE_WAITDO:CURLM_STATE_DO);
           else {
 #ifndef CURL_DISABLE_HTTP
-            if(data->easy_conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)
+            if(Curl_connect_ongoing(data->easy_conn))
               multistate(data, CURLM_STATE_WAITPROXYCONNECT);
             else
 #endif
@@ -1552,7 +1560,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       else if(!result) {
         if((data->easy_conn->http_proxy.proxytype != CURLPROXY_HTTPS ||
            data->easy_conn->bits.proxy_ssl_connected[FIRSTSOCKET]) &&
-           (data->easy_conn->tunnel_state[FIRSTSOCKET] != TUNNEL_CONNECT)) {
+           Curl_connect_complete(data->easy_conn)) {
           rc = CURLM_CALL_MULTI_PERFORM;
           /* initiate protocol connect phase */
           multistate(data, CURLM_STATE_SENDPROTOCONNECT);
@@ -1568,7 +1576,7 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
 #ifndef CURL_DISABLE_HTTP
         if((data->easy_conn->http_proxy.proxytype == CURLPROXY_HTTPS &&
             !data->easy_conn->bits.proxy_ssl_connected[FIRSTSOCKET]) ||
-            (data->easy_conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)) {
+           Curl_connect_ongoing(data->easy_conn)) {
           multistate(data, CURLM_STATE_WAITPROXYCONNECT);
           break;
         }
diff --git a/lib/url.c b/lib/url.c
index 87446dbca..51e50a80b 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -3023,7 +3023,7 @@ static void conn_free(struct connectdata *conn)
   Curl_safefree(conn->http_proxy.host.rawalloc); /* http proxy name buffer */
   Curl_safefree(conn->socks_proxy.host.rawalloc); /* socks proxy name buffer */
   Curl_safefree(conn->master_buffer);
-  Curl_safefree(conn->connect_buffer);
+  Curl_safefree(conn->connect_state);
 
   conn_reset_all_postponed_data(conn);
 
@@ -4013,7 +4013,7 @@ CURLcode Curl_protocol_connect(struct connectdata *conn,
       return CURLE_OK;
 
     if(conn->bits.tunnel_proxy && conn->bits.httpproxy &&
-       (conn->tunnel_state[FIRSTSOCKET] != TUNNEL_COMPLETE))
+       Curl_connect_ongoing(conn))
       /* when using an HTTP tunnel proxy, await complete tunnel establishment
          before proceeding further. Return CURLE_OK so we'll be called again */
       return CURLE_OK;
diff --git a/lib/urldata.h b/lib/urldata.h
index 2d0f46a17..681fef1fb 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -892,6 +892,23 @@ struct proxy_info {
   char *passwd;  /* proxy password string, allocated */
 };
 
+#define CONNECT_BUFFER_SIZE 16384
+
+/* struct for HTTP CONNECT state data */
+struct http_connect_state {
+  char connect_buffer[CONNECT_BUFFER_SIZE];
+  int perline; /* count bytes per line */
+  int keepon;
+  char *line_start;
+  char *ptr; /* where to store more data */
+  curl_off_t cl; /* size of content to read and ignore */
+  enum {
+    TUNNEL_INIT,    /* init/default/no tunnel state */
+    TUNNEL_CONNECT, /* CONNECT has been sent off */
+    TUNNEL_COMPLETE /* CONNECT response received completely */
+  } tunnel_state;
+};
+
 /*
  * The connectdata struct contains all fields and variables that should be
  * unique for an entire connection.
@@ -1134,17 +1151,9 @@ struct connectdata {
   char *localdev;
   unsigned short localport;
   int localportrange;
-
-  /* tunnel as in tunnel through a HTTP proxy with CONNECT */
-  enum {
-    TUNNEL_INIT,    /* init/default/no tunnel state */
-    TUNNEL_CONNECT, /* CONNECT has been sent off */
-    TUNNEL_COMPLETE /* CONNECT response received completely */
-  } tunnel_state[2]; /* two separate ones to allow FTP */
+  struct http_connect_state *connect_state; /* for HTTP CONNECT */
   struct connectbundle *bundle; /* The bundle we are member of */
-
   int negnpn; /* APLN or NPN TLS negotiated protocol, CURL_HTTP_VERSION* */
-  char *connect_buffer; /* for CONNECT business */
 
 #ifdef USE_UNIX_SOCKETS
   char *unix_domain_socket;
diff --git a/tests/data/test1021 b/tests/data/test1021
index 29247f11d..85845a5ba 100644
--- a/tests/data/test1021
+++ b/tests/data/test1021
@@ -15,7 +15,7 @@ HTTP proxy NTLM auth
 <connect>
 HTTP/1.1 407 Authorization Required to proxy me my dear swsclose

 Proxy-Authenticate: NTLM

-Content-Length: 21

+Content-Length: 16

 Connection: close

 
 data to discard
@@ -62,7 +62,7 @@ Nice proxy auth sir!
 <datacheck>
 HTTP/1.1 407 Authorization Required to proxy me my dear swsclose

 Proxy-Authenticate: NTLM

-Content-Length: 21

+Content-Length: 16

 Connection: close

 
 HTTP/1.1 407 Authorization Required to proxy me my dear