http-proxy: move proxy code to http_proxy.c

The new http_proxy.* files now host HTTP proxy specific code (500+ lines
moved out from http.c), and as a consequence there is a macro introduced
for the Curl_proxyCONNECT() function so that code can use it without
actually supporting proxy (or HTTP) in builds.

11 files changed, 585 insertions, 497 deletions

diff --git a/lib/Makefile.inc b/lib/Makefile.inc
index a71900bcf..65f2d7e2b 100644
--- a/lib/Makefile.inc
+++ b/lib/Makefile.inc
@@ -21,7 +21,7 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c	\
   socks_gssapi.c socks_sspi.c curl_sspi.c slist.c nonblock.c		\
   curl_memrchr.c imap.c pop3.c smtp.c pingpong.c rtsp.c curl_threads.c	\
   warnless.c hmac.c polarssl.c curl_rtmp.c openldap.c curl_gethostname.c\
-  gopher.c axtls.c idn_win32.c http_negotiate_sspi.c cyassl.c
+  gopher.c axtls.c idn_win32.c http_negotiate_sspi.c cyassl.c http_proxy.c
 
 HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h	\
   progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h	\
@@ -36,5 +36,5 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h	\
   curl_base64.h rawstr.h curl_addrinfo.h curl_sspi.h slist.h nonblock.h	\
   curl_memrchr.h imap.h pop3.h smtp.h pingpong.h rtsp.h curl_threads.h	\
   warnless.h curl_hmac.h polarssl.h curl_rtmp.h curl_gethostname.h      \
-  gopher.h axtls.h cyassl.h
+  gopher.h axtls.h cyassl.h http_proxy.h
 
diff --git a/lib/Makefile.vc6 b/lib/Makefile.vc6
index eff17d021..cd712306d 100644
--- a/lib/Makefile.vc6
+++ b/lib/Makefile.vc6
@@ -496,6 +496,7 @@ X_OBJS= \
         $(DIROBJ)\http_negotiate_sspi.obj \

 	$(DIROBJ)\http_ntlm.obj \

 	$(DIROBJ)\http.obj \

+	$(DIROBJ)\http_proxy.obj \

 	$(DIROBJ)\if2ip.obj \

 	$(DIROBJ)\imap.obj \

 	$(DIROBJ)\inet_ntop.obj \

diff --git a/lib/ftp.c b/lib/ftp.c
index 6b4381909..8dfdc1dc7 100644
--- a/lib/ftp.c
+++ b/lib/ftp.c
@@ -93,6 +93,7 @@
 #include "rawstr.h"
 #include "speedcheck.h"
 #include "warnless.h"
+#include "http_proxy.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -1744,7 +1745,7 @@ static CURLcode ftp_state_pasv_resp(struct connectdata *conn,
     result = CURLE_COULDNT_CONNECT;
     break;
   }
-#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_PROXY)
+
   if(conn->bits.tunnel_proxy && conn->bits.httpproxy) {
     /* FIX: this MUST wait for a proper connect first if 'connected' is
      * FALSE */
@@ -1770,7 +1771,7 @@ static CURLcode ftp_state_pasv_resp(struct connectdata *conn,
     if(CURLE_OK != result)
       return result;
   }
-#endif /* !CURL_DISABLE_HTTP && !CURL_DISABLE_PROXY */
+
 
   state(conn, FTP_STOP); /* this phase is completed */
 
diff --git a/lib/http.c b/lib/http.c
index 01a78a977..175ec7be7 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -98,6 +98,7 @@
 #include "rawstr.h"
 #include "content_encoding.h"
 #include "rtsp.h"
+#include "http_proxy.h"
 #include "warnless.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
@@ -106,9 +107,6 @@
 /* The last #include file should be: */
 #include "memdebug.h"
 
-/* Default proxy timeout in milliseconds */
-#define PROXY_TIMEOUT (3600*1000)
-
 /*
  * Forward declarations.
  */
@@ -609,12 +607,12 @@ output_auth_headers(struct connectdata *conn,
  *
  * @returns CURLcode
  */
-static CURLcode
-http_output_auth(struct connectdata *conn,
-                 const char *request,
-                 const char *path,
-                 bool proxytunnel) /* TRUE if this is the request setting
-                                      up the proxy tunnel */
+CURLcode
+Curl_http_output_auth(struct connectdata *conn,
+                      const char *request,
+                      const char *path,
+                      bool proxytunnel) /* TRUE if this is the request setting
+                                           up the proxy tunnel */
 {
   CURLcode result = CURLE_OK;
   struct SessionHandle *data = conn->data;
@@ -1270,474 +1268,6 @@ Curl_compareheader(const char *headerline, /* line to check */
   return FALSE; /* no match */
 }
 
-#ifndef CURL_DISABLE_PROXY
-/*
- * Curl_proxyCONNECT() requires that we're connected to a HTTP proxy. This
- * function will issue the necessary commands to get a seamless tunnel through
- * this proxy. After that, the socket can be used just as a normal socket.
- *
- * This badly needs to be rewritten. CONNECT should be sent and dealt with
- * like any ordinary HTTP request, and not specially crafted like this. This
- * function only remains here like this for now since the rewrite is a bit too
- * much work to do at the moment.
- *
- * This function is BLOCKING which is nasty for all multi interface using apps.
- */
-
-CURLcode Curl_proxyCONNECT(struct connectdata *conn,
-                           int sockindex,
-                           const char *hostname,
-                           unsigned short remote_port)
-{
-  int subversion=0;
-  struct SessionHandle *data=conn->data;
-  struct SingleRequest *k = &data->req;
-  CURLcode result;
-  long timeout =
-    data->set.timeout?data->set.timeout:PROXY_TIMEOUT; /* in milliseconds */
-  curl_socket_t tunnelsocket = conn->sock[sockindex];
-  curl_off_t cl=0;
-  bool closeConnection = FALSE;
-  bool chunked_encoding = FALSE;
-  long check;
-
-#define SELECT_OK      0
-#define SELECT_ERROR   1
-#define SELECT_TIMEOUT 2
-  int error = SELECT_OK;
-
-  conn->bits.proxy_connect_closed = FALSE;
-
-  do {
-    if(!conn->bits.tunnel_connecting) { /* BEGIN CONNECT PHASE */
-      char *host_port;
-      Curl_send_buffer *req_buffer;
-
-      infof(data, "Establish HTTP proxy tunnel to %s:%hu\n",
-            hostname, remote_port);
-
-      if(data->req.newurl) {
-        /* This only happens if we've looped here due to authentication
-           reasons, and we don't really use the newly cloned URL here
-           then. Just free() it. */
-        free(data->req.newurl);
-        data->req.newurl = NULL;
-      }
-
-      /* initialize a dynamic send-buffer */
-      req_buffer = Curl_add_buffer_init();
-
-      if(!req_buffer)
-        return CURLE_OUT_OF_MEMORY;
-
-      host_port = aprintf("%s:%hu", hostname, remote_port);
-      if(!host_port) {
-        free(req_buffer);
-        return CURLE_OUT_OF_MEMORY;
-      }
-
-      /* Setup the proxy-authorization header, if any */
-      result = http_output_auth(conn, "CONNECT", host_port, TRUE);
-
-      if(CURLE_OK == result) {
-        char *host=(char *)"";
-        const char *proxyconn="";
-        const char *useragent="";
-        const char *http = (conn->proxytype == CURLPROXY_HTTP_1_0) ?
-          "1.0" : "1.1";
-
-        if(!Curl_checkheaders(data, "Host:")) {
-          host = aprintf("Host: %s\r\n", host_port);
-          if(!host) {
-            free(req_buffer);
-            free(host_port);
-            return CURLE_OUT_OF_MEMORY;
-          }
-        }
-        if(!Curl_checkheaders(data, "Proxy-Connection:"))
-          proxyconn = "Proxy-Connection: Keep-Alive\r\n";
-
-        if(!Curl_checkheaders(data, "User-Agent:") &&
-           data->set.str[STRING_USERAGENT])
-          useragent = conn->allocptr.uagent;
-
-        /* Send the connect request to the proxy */
-        /* BLOCKING */
-        result =
-          Curl_add_bufferf(req_buffer,
-                      "CONNECT %s:%hu HTTP/%s\r\n"
-                      "%s"  /* Host: */
-                      "%s"  /* Proxy-Authorization */
-                      "%s"  /* User-Agent */
-                      "%s", /* Proxy-Connection */
-                      hostname, remote_port, http,
-                      host,
-                      conn->allocptr.proxyuserpwd?
-                      conn->allocptr.proxyuserpwd:"",
-                      useragent,
-                      proxyconn);
-
-        if(host && *host)
-          free(host);
-
-        if(CURLE_OK == result)
-          result = Curl_add_custom_headers(conn, req_buffer);
-
-        if(CURLE_OK == result)
-          /* CRLF terminate the request */
-          result = Curl_add_bufferf(req_buffer, "\r\n");
-
-        if(CURLE_OK == result) {
-          /* Now send off the request */
-          result = Curl_add_buffer_send(req_buffer, conn,
-                                        &data->info.request_size, 0, sockindex);
-        }
-        req_buffer = NULL;
-        if(result)
-          failf(data, "Failed sending CONNECT to proxy");
-      }
-      free(host_port);
-      Curl_safefree(req_buffer);
-      if(result)
-        return result;
-
-      conn->bits.tunnel_connecting = TRUE;
-    } /* END CONNECT PHASE */
-
-    /* now we've issued the CONNECT and we're waiting to hear back -
-       we try not to block here in multi-mode because that might be a LONG
-       wait if the proxy cannot connect-through to the remote host. */
-
-    /* if timeout is requested, find out how much remaining time we have */
-    check = timeout - /* timeout time */
-      Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
-    if(check <= 0) {
-      failf(data, "Proxy CONNECT aborted due to timeout");
-      return CURLE_RECV_ERROR;
-    }
-
-    /* if we're in multi-mode and we would block, return instead for a retry */
-    if(Curl_if_multi == data->state.used_interface) {
-      if(0 == Curl_socket_ready(tunnelsocket, CURL_SOCKET_BAD, 0))
-        /* return so we'll be called again polling-style */
-        return CURLE_OK;
-      else {
-        DEBUGF(infof(data,
-                     "Multi mode finished polling for response from "
-                     "proxy CONNECT."));
-      }
-    }
-    else {
-      DEBUGF(infof(data, "Easy mode waiting response from proxy CONNECT."));
-    }
-
-    /* at this point, either:
-       1) we're in easy-mode and so it's okay to block waiting for a CONNECT
-       response
-       2) we're in multi-mode and we didn't block - it's either an error or we
-       now have some data waiting.
-       In any case, the tunnel_connecting phase is over. */
-    conn->bits.tunnel_connecting = FALSE;
-
-    { /* BEGIN NEGOTIATION PHASE */
-      size_t nread;   /* total size read */
-      int perline; /* count bytes per line */
-      int keepon=TRUE;
-      ssize_t gotbytes;
-      char *ptr;
-      char *line_start;
-
-      ptr=data->state.buffer;
-      line_start = ptr;
-
-      nread=0;
-      perline=0;
-      keepon=TRUE;
-
-      while((nread<BUFSIZE) && (keepon && !error)) {
-
-        /* if timeout is requested, find out how much remaining time we have */
-        check = timeout - /* timeout time */
-          Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
-        if(check <= 0) {
-          failf(data, "Proxy CONNECT aborted due to timeout");
-          error = SELECT_TIMEOUT; /* already too little time */
-          break;
-        }
-
-        /* loop every second at least, less if the timeout is near */
-        switch (Curl_socket_ready(tunnelsocket, CURL_SOCKET_BAD,
-                            check<1000L?(int)check:1000)) {
-        case -1: /* select() error, stop reading */
-          error = SELECT_ERROR;
-          failf(data, "Proxy CONNECT aborted due to select/poll error");
-          break;
-        case 0: /* timeout */
-          break;
-        default:
-          DEBUGASSERT(ptr+BUFSIZE-nread <= data->state.buffer+BUFSIZE+1);
-          result = Curl_read(conn, tunnelsocket, ptr, BUFSIZE-nread, &gotbytes);
-          if(result==CURLE_AGAIN)
-            continue; /* go loop yourself */
-          else if(result)
-            keepon = FALSE;
-          else if(gotbytes <= 0) {
-            keepon = FALSE;
-            if(data->set.proxyauth && data->state.authproxy.avail) {
-              /* proxy auth was requested and there was proxy auth available,
-                 then deem this as "mere" proxy disconnect */
-              conn->bits.proxy_connect_closed = TRUE;
-            }
-            else {
-              error = SELECT_ERROR;
-              failf(data, "Proxy CONNECT aborted");
-            }
-          }
-          else {
-            /*
-             * We got a whole chunk of data, which can be anything from one
-             * byte to a set of lines and possibly just a piece of the last
-             * line.
-             */
-            int i;
-
-            nread += gotbytes;
-
-            if(keepon > TRUE) {
-              /* This means we are currently ignoring a response-body */
-
-              nread = 0; /* make next read start over in the read buffer */
-              ptr=data->state.buffer;
-              if(cl) {
-                /* A Content-Length based body: simply count down the counter
-                   and make sure to break out of the loop when we're done! */
-                cl -= gotbytes;
-                if(cl<=0) {
-                  keepon = FALSE;
-                  break;
-                }
-              }
-              else {
-                /* chunked-encoded body, so we need to do the chunked dance
-                   properly to know when the end of the body is reached */
-                CHUNKcode r;
-                ssize_t tookcareof=0;
-
-                /* now parse the chunked piece of data so that we can
-                   properly tell when the stream ends */
-                r = Curl_httpchunk_read(conn, ptr, gotbytes, &tookcareof);
-                if(r == CHUNKE_STOP) {
-                  /* we're done reading chunks! */
-                  infof(data, "chunk reading DONE\n");
-                  keepon = FALSE;
-                }
-                else
-                  infof(data, "Read %zd bytes of chunk, continue\n",
-                        tookcareof);
-              }
-            }
-            else
-              for(i = 0; i < gotbytes; ptr++, i++) {
-                perline++; /* amount of bytes in this line so far */
-                if(*ptr == 0x0a) {
-                  char letter;
-                  int writetype;
-
-#ifdef CURL_DOES_CONVERSIONS
-                  /* convert from the network encoding */
-                  result = Curl_convert_from_network(data, line_start,
-                                                     perline);
-                  /* Curl_convert_from_network calls failf if unsuccessful */
-                  if(result)
-                    return result;
-#endif /* CURL_DOES_CONVERSIONS */
-
-                  /* output debug if that is requested */
-                  if(data->set.verbose)
-                    Curl_debug(data, CURLINFO_HEADER_IN,
-                               line_start, (size_t)perline, conn);
-
-                  /* send the header to the callback */
-                  writetype = CLIENTWRITE_HEADER;
-                  if(data->set.include_header)
-                    writetype |= CLIENTWRITE_BODY;
-
-                  result = Curl_client_write(conn, writetype, line_start,
-                                             perline);
-                  if(result)
-                    return result;
-
-                  /* Newlines are CRLF, so the CR is ignored as the line isn't
-                     really terminated until the LF comes. Treat a following CR
-                     as end-of-headers as well.*/
-
-                  if(('\r' == line_start[0]) ||
-                     ('\n' == line_start[0])) {
-                    /* end of response-headers from the proxy */
-                    nread = 0; /* make next read start over in the read
-                                  buffer */
-                    ptr=data->state.buffer;
-                    if((407 == k->httpcode) && !data->state.authproblem) {
-                      /* If we get a 407 response code with content length
-                         when we have no auth problem, we must ignore the
-                         whole response-body */
-                      keepon = 2;
-
-                      if(cl) {
-
-                        infof(data, "Ignore %" FORMAT_OFF_T
-                              " bytes of response-body\n", cl);
-                        /* remove the remaining chunk of what we already
-                           read */
-                        cl -= (gotbytes - i);
-
-                        if(cl<=0)
-                          /* if the whole thing was already read, we are done!
-                           */
-                          keepon=FALSE;
-                      }
-                      else if(chunked_encoding) {
-                        CHUNKcode r;
-                        /* We set ignorebody true here since the chunked
-                           decoder function will acknowledge that. Pay
-                           attention so that this is cleared again when this
-                           function returns! */
-                        k->ignorebody = TRUE;
-                        infof(data, "%zd bytes of chunk left\n", gotbytes-i);
-
-                        if(line_start[1] == '\n') {
-                          /* this can only be a LF if the letter at index 0
-                             was a CR */
-                          line_start++;
-                          i++;
-                        }
-
-                        /* now parse the chunked piece of data so that we can
-                           properly tell when the stream ends */
-                        r = Curl_httpchunk_read(conn, line_start+1,
-                                                  gotbytes -i, &gotbytes);
-                        if(r == CHUNKE_STOP) {
-                          /* we're done reading chunks! */
-                          infof(data, "chunk reading DONE\n");
-                          keepon = FALSE;
-                        }
-                        else
-                          infof(data, "Read %zd bytes of chunk, continue\n",
-                                gotbytes);
-                      }
-                      else {
-                        /* without content-length or chunked encoding, we
-                           can't keep the connection alive since the close is
-                           the end signal so we bail out at once instead */
-                        keepon=FALSE;
-                      }
-                    }
-                    else
-                      keepon = FALSE;
-                    break; /* breaks out of for-loop, not switch() */
-                  }
-
-                  /* keep a backup of the position we are about to blank */
-                  letter = line_start[perline];
-                  line_start[perline]=0; /* zero terminate the buffer */
-                  if((checkprefix("WWW-Authenticate:", line_start) &&
-                      (401 == k->httpcode)) ||
-                     (checkprefix("Proxy-authenticate:", line_start) &&
-                      (407 == k->httpcode))) {
-                    result = Curl_http_input_auth(conn, k->httpcode,
-                                                  line_start);
-                    if(result)
-                      return result;
-                  }
-                  else if(checkprefix("Content-Length:", line_start)) {
-                    cl = curlx_strtoofft(line_start +
-                                         strlen("Content-Length:"), NULL, 10);
-                  }
-                  else if(Curl_compareheader(line_start,
-                                             "Connection:", "close"))
-                    closeConnection = TRUE;
-                  else if(Curl_compareheader(line_start,
-                                             "Transfer-Encoding:",
-                                             "chunked")) {
-                    infof(data, "CONNECT responded chunked\n");
-                    chunked_encoding = TRUE;
-                    /* init our chunky engine */
-                    Curl_httpchunk_init(conn);
-                  }
-                  else if(Curl_compareheader(line_start,
-                                             "Proxy-Connection:", "close"))
-                    closeConnection = TRUE;
-                  else if(2 == sscanf(line_start, "HTTP/1.%d %d",
-                                      &subversion,
-                                      &k->httpcode)) {
-                    /* store the HTTP code from the proxy */
-                    data->info.httpproxycode = k->httpcode;
-                  }
-                  /* put back the letter we blanked out before */
-                  line_start[perline]= letter;
-
-                  perline=0; /* line starts over here */
-                  line_start = ptr+1; /* this skips the zero byte we wrote */
-                }
-              }
-          }
-          break;
-        } /* switch */
-        if(Curl_pgrsUpdate(conn))
-          return CURLE_ABORTED_BY_CALLBACK;
-      } /* while there's buffer left and loop is requested */
-
-      if(error)
-        return CURLE_RECV_ERROR;
-
-      if(data->info.httpproxycode != 200) {
-        /* Deal with the possibly already received authenticate
-           headers. 'newurl' is set to a new URL if we must loop. */
-        result = Curl_http_auth_act(conn);
-        if(result)
-          return result;
-
-        if(conn->bits.close)
-          /* the connection has been marked for closure, most likely in the
-             Curl_http_auth_act() function and thus we can kill it at once
-             below
-          */
-          closeConnection = TRUE;
-      }
-
-      if(closeConnection && data->req.newurl) {
-        /* Connection closed by server. Don't use it anymore */
-        sclose(conn->sock[sockindex]);
-        conn->sock[sockindex] = CURL_SOCKET_BAD;
-        break;
-      }
-    } /* END NEGOTIATION PHASE */
-  } while(data->req.newurl);
-
-  if(200 != data->req.httpcode) {
-    failf(data, "Received HTTP code %d from proxy after CONNECT",
-          data->req.httpcode);
-
-    if(closeConnection && data->req.newurl)
-      conn->bits.proxy_connect_closed = TRUE;
-
-    return CURLE_RECV_ERROR;
-  }
-
-  /* If a proxy-authorization header was used for the proxy, then we should
-     make sure that it isn't accidentally used for the document request
-     after we've connected. So let's free and clear it here. */
-  Curl_safefree(conn->allocptr.proxyuserpwd);
-  conn->allocptr.proxyuserpwd = NULL;
-
-  data->state.authproxy.done = TRUE;
-
-  infof (data, "Proxy replied OK to CONNECT request\n");
-  data->req.ignorebody = FALSE; /* put it (back) to non-ignore state */
-  return CURLE_OK;
-}
-#endif /* CURL_DISABLE_PROXY */
-
 /*
  * Curl_http_connect() performs HTTP stuff to do at connect-time, called from
  * the generic Curl_connect().
@@ -2166,7 +1696,7 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
   }
 
   /* setup the authentication headers */
-  result = http_output_auth(conn, request, ppath, FALSE);
+  result = Curl_http_output_auth(conn, request, ppath, FALSE);
   if(result)
     return result;
 
diff --git a/lib/http.h b/lib/http.h
index 3007c319f..fab2bfd25 100644
--- a/lib/http.h
+++ b/lib/http.h
@@ -8,7 +8,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2010, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -66,12 +66,6 @@ CURLcode Curl_add_timecondition(struct SessionHandle *data,
 CURLcode Curl_add_custom_headers(struct connectdata *conn,
                                    Curl_send_buffer *req_buffer);
 
-
-/* ftp can use this as well */
-CURLcode Curl_proxyCONNECT(struct connectdata *conn,
-                           int tunnelsocket,
-                           const char *hostname, unsigned short remote_port);
-
 /* protocol-specific functions set up to be called by the main engine */
 CURLcode Curl_http(struct connectdata *conn, bool *done);
 CURLcode Curl_http_done(struct connectdata *, CURLcode, bool premature);
@@ -158,4 +152,25 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data,
                                      ssize_t *nread,
                                      bool *stop_reading);
 
+/**
+ * Curl_http_output_auth() setups the authentication headers for the
+ * host/proxy and the correct authentication
+ * method. conn->data->state.authdone is set to TRUE when authentication is
+ * done.
+ *
+ * @param conn all information about the current connection
+ * @param request pointer to the request keyword
+ * @param path pointer to the requested path
+ * @param proxytunnel boolean if this is the request setting up a "proxy
+ * tunnel"
+ *
+ * @returns CURLcode
+ */
+CURLcode
+Curl_http_output_auth(struct connectdata *conn,
+                      const char *request,
+                      const char *path,
+                      bool proxytunnel); /* TRUE if this is the request setting
+                                            up the proxy tunnel */
+
 #endif
diff --git a/lib/http_proxy.c b/lib/http_proxy.c
new file mode 100644
index 000000000..7631ae53c
--- /dev/null
+++ b/lib/http_proxy.c
@@ -0,0 +1,511 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at http://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+
+#include "setup.h"
+
+#if !defined(CURL_DISABLE_PROXY) && !defined(CURL_DISABLE_HTTP)
+
+#include "urldata.h"
+#include <curl/curl.h>
+#include "http_proxy.h"
+#include "sendf.h"
+#include "http.h"
+#include "url.h"
+#include "select.h"
+#include "rawstr.h"
+#include "progress.h"
+
+#define _MPRINTF_REPLACE /* use our functions only */
+#include <curl/mprintf.h>
+
+#include "curlx.h"
+
+/* The last #include file should be: */
+#include "memdebug.h"
+
+/*
+ * Curl_proxyCONNECT() requires that we're connected to a HTTP proxy. This
+ * function will issue the necessary commands to get a seamless tunnel through
+ * this proxy. After that, the socket can be used just as a normal socket.
+ *
+ * This badly needs to be rewritten. CONNECT should be sent and dealt with
+ * like any ordinary HTTP request, and not specially crafted like this. This
+ * function only remains here like this for now since the rewrite is a bit too
+ * much work to do at the moment.
+ *
+ * This function is BLOCKING which is nasty for all multi interface using apps.
+ */
+
+CURLcode Curl_proxyCONNECT(struct connectdata *conn,
+                           int sockindex,
+                           const char *hostname,
+                           unsigned short remote_port)
+{
+  int subversion=0;
+  struct SessionHandle *data=conn->data;
+  struct SingleRequest *k = &data->req;
+  CURLcode result;
+  long timeout =
+    data->set.timeout?data->set.timeout:PROXY_TIMEOUT; /* in milliseconds */
+  curl_socket_t tunnelsocket = conn->sock[sockindex];
+  curl_off_t cl=0;
+  bool closeConnection = FALSE;
+  bool chunked_encoding = FALSE;
+  long check;
+
+#define SELECT_OK      0
+#define SELECT_ERROR   1
+#define SELECT_TIMEOUT 2
+  int error = SELECT_OK;
+
+  conn->bits.proxy_connect_closed = FALSE;
+
+  do {
+    if(!conn->bits.tunnel_connecting) { /* BEGIN CONNECT PHASE */
+      char *host_port;
+      Curl_send_buffer *req_buffer;
+
+      infof(data, "Establish HTTP proxy tunnel to %s:%hu\n",
+            hostname, remote_port);
+
+      if(data->req.newurl) {
+        /* This only happens if we've looped here due to authentication
+           reasons, and we don't really use the newly cloned URL here
+           then. Just free() it. */
+        free(data->req.newurl);
+        data->req.newurl = NULL;
+      }
+
+      /* initialize a dynamic send-buffer */
+      req_buffer = Curl_add_buffer_init();
+
+      if(!req_buffer)
+        return CURLE_OUT_OF_MEMORY;
+
+      host_port = aprintf("%s:%hu", hostname, remote_port);
+      if(!host_port) {
+        free(req_buffer);
+        return CURLE_OUT_OF_MEMORY;
+      }
+
+      /* Setup the proxy-authorization header, if any */
+      result = Curl_http_output_auth(conn, "CONNECT", host_port, TRUE);
+
+      if(CURLE_OK == result) {
+        char *host=(char *)"";
+        const char *proxyconn="";
+        const char *useragent="";
+        const char *http = (conn->proxytype == CURLPROXY_HTTP_1_0) ?
+          "1.0" : "1.1";
+
+        if(!Curl_checkheaders(data, "Host:")) {
+          host = aprintf("Host: %s\r\n", host_port);
+          if(!host) {
+            free(req_buffer);
+            free(host_port);
+            return CURLE_OUT_OF_MEMORY;
+          }
+        }
+        if(!Curl_checkheaders(data, "Proxy-Connection:"))
+          proxyconn = "Proxy-Connection: Keep-Alive\r\n";
+
+        if(!Curl_checkheaders(data, "User-Agent:") &&
+           data->set.str[STRING_USERAGENT])
+          useragent = conn->allocptr.uagent;
+
+        /* Send the connect request to the proxy */
+        /* BLOCKING */
+        result =
+          Curl_add_bufferf(req_buffer,
+                      "CONNECT %s:%hu HTTP/%s\r\n"
+                      "%s"  /* Host: */
+                      "%s"  /* Proxy-Authorization */
+                      "%s"  /* User-Agent */
+                      "%s", /* Proxy-Connection */
+                      hostname, remote_port, http,
+                      host,
+                      conn->allocptr.proxyuserpwd?
+                      conn->allocptr.proxyuserpwd:"",
+                      useragent,
+                      proxyconn);
+
+        if(host && *host)
+          free(host);
+
+        if(CURLE_OK == result)
+          result = Curl_add_custom_headers(conn, req_buffer);
+
+        if(CURLE_OK == result)
+          /* CRLF terminate the request */
+          result = Curl_add_bufferf(req_buffer, "\r\n");
+
+        if(CURLE_OK == result) {
+          /* Now send off the request */
+          result = Curl_add_buffer_send(req_buffer, conn,
+                                        &data->info.request_size, 0, sockindex);
+        }
+        req_buffer = NULL;
+        if(result)
+          failf(data, "Failed sending CONNECT to proxy");
+      }
+      free(host_port);
+      Curl_safefree(req_buffer);
+      if(result)
+        return result;
+
+      conn->bits.tunnel_connecting = TRUE;
+    } /* END CONNECT PHASE */
+
+    /* now we've issued the CONNECT and we're waiting to hear back -
+       we try not to block here in multi-mode because that might be a LONG
+       wait if the proxy cannot connect-through to the remote host. */
+
+    /* if timeout is requested, find out how much remaining time we have */
+    check = timeout - /* timeout time */
+      Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
+    if(check <= 0) {
+      failf(data, "Proxy CONNECT aborted due to timeout");
+      return CURLE_RECV_ERROR;
+    }
+
+    /* if we're in multi-mode and we would block, return instead for a retry */
+    if(Curl_if_multi == data->state.used_interface) {
+      if(0 == Curl_socket_ready(tunnelsocket, CURL_SOCKET_BAD, 0))
+        /* return so we'll be called again polling-style */
+        return CURLE_OK;
+      else {
+        DEBUGF(infof(data,
+                     "Multi mode finished polling for response from "
+                     "proxy CONNECT."));
+      }
+    }
+    else {
+      DEBUGF(infof(data, "Easy mode waiting response from proxy CONNECT."));
+    }
+
+    /* at this point, either:
+       1) we're in easy-mode and so it's okay to block waiting for a CONNECT
+       response
+       2) we're in multi-mode and we didn't block - it's either an error or we
+       now have some data waiting.
+       In any case, the tunnel_connecting phase is over. */
+    conn->bits.tunnel_connecting = FALSE;
+
+    { /* BEGIN NEGOTIATION PHASE */
+      size_t nread;   /* total size read */
+      int perline; /* count bytes per line */
+      int keepon=TRUE;
+      ssize_t gotbytes;
+      char *ptr;
+      char *line_start;
+
+      ptr=data->state.buffer;
+      line_start = ptr;
+
+      nread=0;
+      perline=0;
+      keepon=TRUE;
+
+      while((nread<BUFSIZE) && (keepon && !error)) {
+
+        /* if timeout is requested, find out how much remaining time we have */
+        check = timeout - /* timeout time */
+          Curl_tvdiff(Curl_tvnow(), conn->now); /* spent time */
+        if(check <= 0) {
+          failf(data, "Proxy CONNECT aborted due to timeout");
+          error = SELECT_TIMEOUT; /* already too little time */
+          break;
+        }
+
+        /* loop every second at least, less if the timeout is near */
+        switch (Curl_socket_ready(tunnelsocket, CURL_SOCKET_BAD,
+                            check<1000L?(int)check:1000)) {
+        case -1: /* select() error, stop reading */
+          error = SELECT_ERROR;
+          failf(data, "Proxy CONNECT aborted due to select/poll error");
+          break;
+        case 0: /* timeout */
+          break;
+        default:
+          DEBUGASSERT(ptr+BUFSIZE-nread <= data->state.buffer+BUFSIZE+1);
+          result = Curl_read(conn, tunnelsocket, ptr, BUFSIZE-nread,
+                             &gotbytes);
+          if(result==CURLE_AGAIN)
+            continue; /* go loop yourself */
+          else if(result)
+            keepon = FALSE;
+          else if(gotbytes <= 0) {
+            keepon = FALSE;
+            if(data->set.proxyauth && data->state.authproxy.avail) {
+              /* proxy auth was requested and there was proxy auth available,
+                 then deem this as "mere" proxy disconnect */
+              conn->bits.proxy_connect_closed = TRUE;
+            }
+            else {
+              error = SELECT_ERROR;
+              failf(data, "Proxy CONNECT aborted");
+            }
+          }
+          else {
+            /*
+             * We got a whole chunk of data, which can be anything from one
+             * byte to a set of lines and possibly just a piece of the last
+             * line.
+             */
+            int i;
+
+            nread += gotbytes;
+
+            if(keepon > TRUE) {
+              /* This means we are currently ignoring a response-body */
+
+              nread = 0; /* make next read start over in the read buffer */
+              ptr=data->state.buffer;
+              if(cl) {
+                /* A Content-Length based body: simply count down the counter
+                   and make sure to break out of the loop when we're done! */
+                cl -= gotbytes;
+                if(cl<=0) {
+                  keepon = FALSE;
+                  break;
+                }
+              }
+              else {
+                /* chunked-encoded body, so we need to do the chunked dance
+                   properly to know when the end of the body is reached */
+                CHUNKcode r;
+                ssize_t tookcareof=0;
+
+                /* now parse the chunked piece of data so that we can
+                   properly tell when the stream ends */
+                r = Curl_httpchunk_read(conn, ptr, gotbytes, &tookcareof);
+                if(r == CHUNKE_STOP) {
+                  /* we're done reading chunks! */
+                  infof(data, "chunk reading DONE\n");
+                  keepon = FALSE;
+                }
+                else
+                  infof(data, "Read %zd bytes of chunk, continue\n",
+                        tookcareof);
+              }
+            }
+            else
+              for(i = 0; i < gotbytes; ptr++, i++) {
+                perline++; /* amount of bytes in this line so far */
+                if(*ptr == 0x0a) {
+                  char letter;
+                  int writetype;
+
+#ifdef CURL_DOES_CONVERSIONS
+                  /* convert from the network encoding */
+                  result = Curl_convert_from_network(data, line_start,
+                                                     perline);
+                  /* Curl_convert_from_network calls failf if unsuccessful */
+                  if(result)
+                    return result;
+#endif /* CURL_DOES_CONVERSIONS */
+
+                  /* output debug if that is requested */
+                  if(data->set.verbose)
+                    Curl_debug(data, CURLINFO_HEADER_IN,
+                               line_start, (size_t)perline, conn);
+
+                  /* send the header to the callback */
+                  writetype = CLIENTWRITE_HEADER;
+                  if(data->set.include_header)
+                    writetype |= CLIENTWRITE_BODY;
+
+                  result = Curl_client_write(conn, writetype, line_start,
+                                             perline);
+                  if(result)
+                    return result;
+
+                  /* Newlines are CRLF, so the CR is ignored as the line isn't
+                     really terminated until the LF comes. Treat a following CR
+                     as end-of-headers as well.*/
+
+                  if(('\r' == line_start[0]) ||
+                     ('\n' == line_start[0])) {
+                    /* end of response-headers from the proxy */
+                    nread = 0; /* make next read start over in the read
+                                  buffer */
+                    ptr=data->state.buffer;
+                    if((407 == k->httpcode) && !data->state.authproblem) {
+                      /* If we get a 407 response code with content length
+                         when we have no auth problem, we must ignore the
+                         whole response-body */
+                      keepon = 2;
+
+                      if(cl) {
+
+                        infof(data, "Ignore %" FORMAT_OFF_T
+                              " bytes of response-body\n", cl);
+                        /* remove the remaining chunk of what we already
+                           read */
+                        cl -= (gotbytes - i);
+
+                        if(cl<=0)
+                          /* if the whole thing was already read, we are done!
+                           */
+                          keepon=FALSE;
+                      }
+                      else if(chunked_encoding) {
+                        CHUNKcode r;
+                        /* We set ignorebody true here since the chunked
+                           decoder function will acknowledge that. Pay
+                           attention so that this is cleared again when this
+                           function returns! */
+                        k->ignorebody = TRUE;
+                        infof(data, "%zd bytes of chunk left\n", gotbytes-i);
+
+                        if(line_start[1] == '\n') {
+                          /* this can only be a LF if the letter at index 0
+                             was a CR */
+                          line_start++;
+                          i++;
+                        }
+
+                        /* now parse the chunked piece of data so that we can
+                           properly tell when the stream ends */
+                        r = Curl_httpchunk_read(conn, line_start+1,
+                                                  gotbytes -i, &gotbytes);
+                        if(r == CHUNKE_STOP) {
+                          /* we're done reading chunks! */
+                          infof(data, "chunk reading DONE\n");
+                          keepon = FALSE;
+                        }
+                        else
+                          infof(data, "Read %zd bytes of chunk, continue\n",
+                                gotbytes);
+                      }
+                      else {
+                        /* without content-length or chunked encoding, we
+                           can't keep the connection alive since the close is
+                           the end signal so we bail out at once instead */
+                        keepon=FALSE;
+                      }
+                    }
+                    else
+                      keepon = FALSE;
+                    break; /* breaks out of for-loop, not switch() */
+                  }
+
+                  /* keep a backup of the position we are about to blank */
+                  letter = line_start[perline];
+                  line_start[perline]=0; /* zero terminate the buffer */
+                  if((checkprefix("WWW-Authenticate:", line_start) &&
+                      (401 == k->httpcode)) ||
+                     (checkprefix("Proxy-authenticate:", line_start) &&
+                      (407 == k->httpcode))) {
+                    result = Curl_http_input_auth(conn, k->httpcode,
+                                                  line_start);
+                    if(result)
+                      return result;
+                  }
+                  else if(checkprefix("Content-Length:", line_start)) {
+                    cl = curlx_strtoofft(line_start +
+                                         strlen("Content-Length:"), NULL, 10);
+                  }
+                  else if(Curl_compareheader(line_start,
+                                             "Connection:", "close"))
+                    closeConnection = TRUE;
+                  else if(Curl_compareheader(line_start,
+                                             "Transfer-Encoding:",
+                                             "chunked")) {
+                    infof(data, "CONNECT responded chunked\n");
+                    chunked_encoding = TRUE;
+                    /* init our chunky engine */
+                    Curl_httpchunk_init(conn);
+                  }
+                  else if(Curl_compareheader(line_start,
+                                             "Proxy-Connection:", "close"))
+                    closeConnection = TRUE;
+                  else if(2 == sscanf(line_start, "HTTP/1.%d %d",
+                                      &subversion,
+                                      &k->httpcode)) {
+                    /* store the HTTP code from the proxy */
+                    data->info.httpproxycode = k->httpcode;
+                  }
+                  /* put back the letter we blanked out before */
+                  line_start[perline]= letter;
+
+                  perline=0; /* line starts over here */
+                  line_start = ptr+1; /* this skips the zero byte we wrote */
+                }
+              }
+          }
+          break;
+        } /* switch */
+        if(Curl_pgrsUpdate(conn))
+          return CURLE_ABORTED_BY_CALLBACK;
+      } /* while there's buffer left and loop is requested */
+
+      if(error)
+        return CURLE_RECV_ERROR;
+
+      if(data->info.httpproxycode != 200) {
+        /* Deal with the possibly already received authenticate
+           headers. 'newurl' is set to a new URL if we must loop. */
+        result = Curl_http_auth_act(conn);
+        if(result)
+          return result;
+
+        if(conn->bits.close)
+          /* the connection has been marked for closure, most likely in the
+             Curl_http_auth_act() function and thus we can kill it at once
+             below
+          */
+          closeConnection = TRUE;
+      }
+
+      if(closeConnection && data->req.newurl) {
+        /* Connection closed by server. Don't use it anymore */
+        sclose(conn->sock[sockindex]);
+        conn->sock[sockindex] = CURL_SOCKET_BAD;
+        break;
+      }
+    } /* END NEGOTIATION PHASE */
+  } while(data->req.newurl);
+
+  if(200 != data->req.httpcode) {
+    failf(data, "Received HTTP code %d from proxy after CONNECT",
+          data->req.httpcode);
+
+    if(closeConnection && data->req.newurl)
+      conn->bits.proxy_connect_closed = TRUE;
+
+    return CURLE_RECV_ERROR;
+  }
+
+  /* If a proxy-authorization header was used for the proxy, then we should
+     make sure that it isn't accidentally used for the document request
+     after we've connected. So let's free and clear it here. */
+  Curl_safefree(conn->allocptr.proxyuserpwd);
+  conn->allocptr.proxyuserpwd = NULL;
+
+  data->state.authproxy.done = TRUE;
+
+  infof (data, "Proxy replied OK to CONNECT request\n");
+  data->req.ignorebody = FALSE; /* put it (back) to non-ignore state */
+  return CURLE_OK;
+}
+#endif /* CURL_DISABLE_PROXY */
diff --git a/lib/http_proxy.h b/lib/http_proxy.h
new file mode 100644
index 000000000..da08fa50a
--- /dev/null
+++ b/lib/http_proxy.h
@@ -0,0 +1,33 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at http://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#if !defined(CURL_DISABLE_PROXY) && !defined(CURL_DISABLE_HTTP)
+/* ftp can use this as well */
+CURLcode Curl_proxyCONNECT(struct connectdata *conn,
+                           int tunnelsocket,
+                           const char *hostname, unsigned short remote_port);
+
+/* Default proxy timeout in milliseconds */
+#define PROXY_TIMEOUT (3600*1000)
+
+#else
+#define Curl_proxyCONNECT(x,y,x,w) CURLE_FAILED_INIT
+#endif
diff --git a/lib/imap.c b/lib/imap.c
index c0406ea42..8f6efc427 100644
--- a/lib/imap.c
+++ b/lib/imap.c
@@ -85,6 +85,7 @@
 #include "url.h"
 #include "rawstr.h"
 #include "strtoofft.h"
+#include "http_proxy.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -718,7 +719,6 @@ static CURLcode imap_connect(struct connectdata *conn,
   pp->endofresp = imap_endofresp;
   pp->conn = conn;
 
-#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_PROXY)
   if(conn->bits.tunnel_proxy && conn->bits.httpproxy) {
     /* for IMAP over HTTP proxy */
     struct HTTP http_proxy;
@@ -745,7 +745,6 @@ static CURLcode imap_connect(struct connectdata *conn,
     if(CURLE_OK != result)
       return result;
   }
-#endif /* !CURL_DISABLE_HTTP && !CURL_DISABLE_PROXY */
 
   if((conn->handler->protocol & CURLPROTO_IMAPS) &&
      data->state.used_interface != Curl_if_multi) {
diff --git a/lib/openldap.c b/lib/openldap.c
index dd008800a..3cb796380 100644
--- a/lib/openldap.c
+++ b/lib/openldap.c
@@ -6,6 +6,7 @@
  *                 \___|\___/|_| \_\_____|
  *
  * Copyright (C) 2010, Howard Chu, <hyc@openldap.org>
+ * Copyright (C) 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -45,6 +46,7 @@
 #include "curl_ldap.h"
 #include "curl_memory.h"
 #include "curl_base64.h"
+#include "http_proxy.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -201,7 +203,6 @@ static CURLcode ldap_connect(struct connectdata *conn, bool *done)
 
   ldap_set_option(li->ld, LDAP_OPT_PROTOCOL_VERSION, &proto);
 
-#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_PROXY)
   if(conn->bits.tunnel_proxy && conn->bits.httpproxy) {
     /* for LDAP over HTTP proxy */
     struct HTTP http_proxy;
@@ -228,7 +229,6 @@ static CURLcode ldap_connect(struct connectdata *conn, bool *done)
     if(CURLE_OK != result)
       return result;
   }
-#endif /* !CURL_DISABLE_HTTP && !CURL_DISABLE_PROXY */
 
 #ifdef USE_SSL
   if (conn->handler->flags & PROTOPT_SSL) {
diff --git a/lib/pop3.c b/lib/pop3.c
index 7f97722a8..a7657171c 100644
--- a/lib/pop3.c
+++ b/lib/pop3.c
@@ -86,6 +86,7 @@
 #include "url.h"
 #include "rawstr.h"
 #include "strtoofft.h"
+#include "http_proxy.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -647,7 +648,6 @@ static CURLcode pop3_connect(struct connectdata *conn,
   pp->endofresp = pop3_endofresp;
   pp->conn = conn;
 
-#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_PROXY)
   if(conn->bits.tunnel_proxy && conn->bits.httpproxy) {
     /* for POP3 over HTTP proxy */
     struct HTTP http_proxy;
@@ -674,7 +674,6 @@ static CURLcode pop3_connect(struct connectdata *conn,
     if(CURLE_OK != result)
       return result;
   }
-#endif /* !CURL_DISABLE_HTTP && !CURL_DISABLE_PROXY */
 
   if(conn->handler->protocol & CURLPROTO_POP3S) {
     /* BLOCKING */
diff --git a/lib/smtp.c b/lib/smtp.c
index 3b2179677..1680a82e1 100644
--- a/lib/smtp.c
+++ b/lib/smtp.c
@@ -93,6 +93,7 @@
 #include "curl_hmac.h"
 #include "curl_gethostname.h"
 #include "warnless.h"
+#include "http_proxy.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -1117,7 +1118,6 @@ static CURLcode smtp_connect(struct connectdata *conn,
   pp->endofresp = smtp_endofresp;
   pp->conn = conn;
 
-#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_PROXY)
   if(conn->bits.tunnel_proxy && conn->bits.httpproxy) {
     /* for SMTP over HTTP proxy */
     struct HTTP http_proxy;
@@ -1144,7 +1144,6 @@ static CURLcode smtp_connect(struct connectdata *conn,
     if(CURLE_OK != result)
       return result;
   }
-#endif /* !CURL_DISABLE_HTTP && !CURL_DISABLE_PROXY */
 
   if((conn->handler->protocol & CURLPROTO_SMTPS) &&
       data->state.used_interface != Curl_if_multi) {