known bug #46: chunked-encoded CONNECT responses from a http proxy now works.

Added test case 1008 to verify. Note that #47 is still there.

4 files changed, 166 insertions, 92 deletions

diff --git a/lib/http.c b/lib/http.c
index 67b2d3f55..90f56f1ba 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -1143,6 +1143,7 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
   curl_socket_t tunnelsocket = conn->sock[sockindex];
   curl_off_t cl=0;
   bool closeConnection = FALSE;
+  bool chunked_encoding = FALSE;
   long check;
 
 #define SELECT_OK      0
@@ -1203,37 +1204,37 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
            data->set.str[STRING_USERAGENT])
           useragent = conn->allocptr.uagent;
 
-	/* Send the connect request to the proxy */
-	/* BLOCKING */
-	result =
-	  add_bufferf(req_buffer,
-		      "CONNECT %s:%d HTTP/1.0\r\n"
-		      "%s"  /* Host: */
-		      "%s"  /* Proxy-Authorization */
-		      "%s"  /* User-Agent */
-		      "%s", /* Proxy-Connection */
-		      hostname, remote_port,
-		      host,
-		      conn->allocptr.proxyuserpwd?
-		      conn->allocptr.proxyuserpwd:"",
-		      useragent,
-		      proxyconn);
-
-	if(host && *host)
-	  free(host);
-
-	if(CURLE_OK == result)
-	  result = add_custom_headers(conn, req_buffer);
-
-	if(CURLE_OK == result)
-	  /* CRLF terminate the request */
-	  result = add_bufferf(req_buffer, "\r\n");
-
-	if(CURLE_OK == result) {
-	  /* Now send off the request */
-	  result = add_buffer_send(req_buffer, conn,
-				   &data->info.request_size, 0, sockindex);
-	}
+        /* Send the connect request to the proxy */
+        /* BLOCKING */
+        result =
+          add_bufferf(req_buffer,
+                      "CONNECT %s:%d HTTP/1.0\r\n"
+                      "%s"  /* Host: */
+                      "%s"  /* Proxy-Authorization */
+                      "%s"  /* User-Agent */
+                      "%s", /* Proxy-Connection */
+                      hostname, remote_port,
+                      host,
+                      conn->allocptr.proxyuserpwd?
+                      conn->allocptr.proxyuserpwd:"",
+                      useragent,
+                      proxyconn);
+
+        if(host && *host)
+          free(host);
+
+        if(CURLE_OK == result)
+          result = add_custom_headers(conn, req_buffer);
+
+        if(CURLE_OK == result)
+          /* CRLF terminate the request */
+          result = add_bufferf(req_buffer, "\r\n");
+
+        if(CURLE_OK == result) {
+          /* Now send off the request */
+          result = add_buffer_send(req_buffer, conn,
+                                   &data->info.request_size, 0, sockindex);
+        }
         req_buffer = NULL;
         if(result)
           failf(data, "Failed sending CONNECT to proxy");
@@ -1340,13 +1341,33 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
             nread += gotbytes;
 
             if(keepon > TRUE) {
-              /* This means we are currently ignoring a response-body, so we
-                 simply count down our counter and make sure to break out of
-                 the loop when we're done! */
-              cl -= gotbytes;
-              if(cl<=0) {
-                keepon = FALSE;
-                break;
+              /* This means we are currently ignoring a response-body */
+              if(cl) {
+                /* A Content-Length based body: simply count down the counter
+                   and make sure to break out of the loop when we're done! */
+                cl -= gotbytes;
+                if(cl<=0) {
+                  keepon = FALSE;
+                  break;
+                }
+              }
+              else {
+                /* chunked-encoded body, so we need to do the chunked dance
+                   properly to know when the end of the body is reached */
+                CHUNKcode r;
+                ssize_t tookcareof=0;
+
+                /* now parse the chunked piece of data so that we can
+                   properly tell when the stream ends */
+                r = Curl_httpchunk_read(conn, ptr, gotbytes, &tookcareof);
+                if(r == CHUNKE_STOP) {
+                  /* we're done reading chunks! */
+                  infof(data, "chunk reading DONE\n");
+                  keepon = FALSE;
+                }
+                else
+                  infof(data, "Read %d bytes of chunk, continue\n",
+                        tookcareof);
               }
             }
             else
@@ -1366,7 +1387,8 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                   if(data->set.include_header)
                     writetype |= CLIENTWRITE_BODY;
 
-                  result = Curl_client_write(conn, writetype, line_start, perline);
+                  result = Curl_client_write(conn, writetype, line_start,
+                                             perline);
                   if(result)
                     return result;
 
@@ -1377,19 +1399,60 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                   if(('\r' == line_start[0]) ||
                      ('\n' == line_start[0])) {
                     /* end of response-headers from the proxy */
-                    if(cl && (407 == k->httpcode) &&
-                       !data->state.authproblem) {
+                    if((407 == k->httpcode) && !data->state.authproblem) {
                       /* If we get a 407 response code with content length
-                       * when we have no auth problem, we must ignore the
-                       * whole response-body */
+                         when we have no auth problem, we must ignore the
+                         whole response-body */
                       keepon = 2;
-                      infof(data, "Ignore %" FORMAT_OFF_T
-                            " bytes of response-body\n", cl);
-                      cl -= (gotbytes - i);/* remove the remaining chunk of
-                                              what we already read */
-                      if(cl<=0)
-                        /* if the whole thing was already read, we are done! */
+
+                      if(cl) {
+
+                        infof(data, "Ignore %" FORMAT_OFF_T
+                              " bytes of response-body\n", cl);
+                        /* remove the remaining chunk of what we already
+                           read */
+                        cl -= (gotbytes - i);
+
+                        if(cl<=0)
+                          /* if the whole thing was already read, we are done!
+                           */
+                          keepon=FALSE;
+                      }
+                      else if(chunked_encoding) {
+                        CHUNKcode r;
+                        /* We set ignorebody true here since the chunked
+                           decoder function will acknowledge that. Pay
+                           attention so that this is cleared again when this
+                           function returns! */
+                        k->ignorebody = TRUE;
+                        infof(data, "%d bytes of chunk left\n", gotbytes-i);
+
+                        if(line_start[1] == '\n') {
+                          /* this can only be a LF if the letter at index 0
+                             was a CR */
+                          line_start++;
+                          i++;
+                        }
+
+                        /* now parse the chunked piece of data so that we can
+                           properly tell when the stream ends */
+                        r = Curl_httpchunk_read(conn, line_start+1,
+                                                  gotbytes -i, &gotbytes);
+                        if(r == CHUNKE_STOP) {
+                          /* we're done reading chunks! */
+                          infof(data, "chunk reading DONE\n");
+                          keepon = FALSE;
+                        }
+                        else
+                          infof(data, "Read %d bytes of chunk, continue\n",
+                                gotbytes);
+                      }
+                      else {
+                        /* without content-length or chunked encoding, we
+                           can't keep the connection alive since the close is
+                           the end signal so we bail out at once instead */
                         keepon=FALSE;
+                      }
                     }
                     else
                       keepon = FALSE;
@@ -1416,6 +1479,13 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                                              "Connection:", "close"))
                     closeConnection = TRUE;
                   else if(Curl_compareheader(line_start,
+                                             "Transfer-Encoding:", "chunked")) {
+                    infof(data, "CONNECT responded chunked\n");
+                    chunked_encoding = TRUE;
+                    /* init our chunky engine */
+                    Curl_httpchunk_init(conn);
+                  }
+                  else if(Curl_compareheader(line_start,
                                              "Proxy-Connection:", "close"))
                     closeConnection = TRUE;
                   else if(2 == sscanf(line_start, "HTTP/1.%d %d",
@@ -1472,6 +1542,7 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
   data->state.authproxy.done = TRUE;
 
   infof (data, "Proxy replied OK to CONNECT request\n");
+  k->ignorebody = FALSE; /* put it (back) to non-ignore state */
   return CURLE_OK;
 }
 
diff --git a/lib/http_chunks.c b/lib/http_chunks.c
index 8e44b6d05..3ec89b9c2 100644
--- a/lib/http_chunks.c
+++ b/lib/http_chunks.c
@@ -84,7 +84,7 @@
 
 void Curl_httpchunk_init(struct connectdata *conn)
 {
-  struct Curl_chunker *chunk = &conn->data->reqdata.proto.http->chunk;
+  struct Curl_chunker *chunk = &conn->chunk;
   chunk->hexindex=0; /* start at 0 */
   chunk->dataleft=0; /* no data left yet! */
   chunk->state = CHUNK_HEX; /* we get hex first! */
@@ -108,7 +108,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 {
   CURLcode result=CURLE_OK;
   struct SessionHandle *data = conn->data;
-  struct Curl_chunker *ch = &data->reqdata.proto.http->chunk;
+  struct Curl_chunker *ch = &conn->chunk;
   struct Curl_transfer_keeper *k = &data->reqdata.keep;
   size_t piece;
   size_t length = (size_t)datalen;
@@ -124,11 +124,11 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
   while(length) {
     switch(ch->state) {
     case CHUNK_HEX:
-       /* Check for an ASCII hex digit.
-          We avoid the use of isxdigit to accommodate non-ASCII hosts. */
-       if((*datap >= 0x30 && *datap <= 0x39)    /* 0-9 */
-       || (*datap >= 0x41 && *datap <= 0x46)    /* A-F */
-       || (*datap >= 0x61 && *datap <= 0x66)) { /* a-f */
+      /* Check for an ASCII hex digit.
+         We avoid the use of isxdigit to accommodate non-ASCII hosts. */
+      if((*datap >= 0x30 && *datap <= 0x39)    /* 0-9 */
+         || (*datap >= 0x41 && *datap <= 0x46)    /* A-F */
+         || (*datap >= 0x61 && *datap <= 0x66)) { /* a-f */
         if(ch->hexindex < MAXNUM_SIZE) {
           ch->hexbuffer[ch->hexindex] = *datap;
           datap++;
@@ -218,39 +218,39 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 #ifdef HAVE_LIBZ
       switch (conn->data->set.http_ce_skip?
               IDENTITY : data->reqdata.keep.content_encoding) {
-        case IDENTITY:
+      case IDENTITY:
 #endif
-          if(!k->ignorebody) {
-            if ( !data->set.http_te_skip )
-              result = Curl_client_write(conn, CLIENTWRITE_BODY, datap,
-                                         piece);
-            else
-              result = CURLE_OK;
-          }
+        if(!k->ignorebody) {
+          if ( !data->set.http_te_skip )
+            result = Curl_client_write(conn, CLIENTWRITE_BODY, datap,
+                                       piece);
+          else
+            result = CURLE_OK;
+        }
 #ifdef HAVE_LIBZ
-          break;
-
-        case DEFLATE:
-          /* update data->reqdata.keep.str to point to the chunk data. */
-          data->reqdata.keep.str = datap;
-          result = Curl_unencode_deflate_write(conn, &data->reqdata.keep,
-                                               (ssize_t)piece);
-          break;
-
-        case GZIP:
-          /* update data->reqdata.keep.str to point to the chunk data. */
-          data->reqdata.keep.str = datap;
-          result = Curl_unencode_gzip_write(conn, &data->reqdata.keep,
-                                            (ssize_t)piece);
-          break;
-
-        case COMPRESS:
-        default:
-          failf (conn->data,
-                 "Unrecognized content encoding type. "
-                 "libcurl understands `identity', `deflate' and `gzip' "
-                 "content encodings.");
-          return CHUNKE_BAD_ENCODING;
+        break;
+
+      case DEFLATE:
+        /* update data->reqdata.keep.str to point to the chunk data. */
+        data->reqdata.keep.str = datap;
+        result = Curl_unencode_deflate_write(conn, &data->reqdata.keep,
+                                             (ssize_t)piece);
+        break;
+
+      case GZIP:
+        /* update data->reqdata.keep.str to point to the chunk data. */
+        data->reqdata.keep.str = datap;
+        result = Curl_unencode_gzip_write(conn, &data->reqdata.keep,
+                                          (ssize_t)piece);
+        break;
+
+      case COMPRESS:
+      default:
+        failf (conn->data,
+               "Unrecognized content encoding type. "
+               "libcurl understands `identity', `deflate' and `gzip' "
+               "content encodings.");
+        return CHUNKE_BAD_ENCODING;
       }
 #endif
 
@@ -319,7 +319,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       else {
         datap++;
         length--;
-     }
+      }
       break;
 
     case CHUNK_TRAILER_CR:
@@ -403,7 +403,6 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
         return CHUNKE_BAD_CHUNK;
       }
 
-
     default:
       return CHUNKE_STATE_ERROR;
     }
diff --git a/lib/transfer.c b/lib/transfer.c
index 7fa88040e..831cb3e98 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -1262,7 +1262,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  We DO care about this data if we are pipelining.
                  Push it back to be read on the next pass. */
 
-              dataleft = data->reqdata.proto.http->chunk.dataleft;
+              dataleft = conn->chunk.dataleft;
               if (dataleft != 0) {
                 infof(conn->data, "Leftovers after chunking. "
                       " Rewinding %d bytes\n",dataleft);
@@ -1617,7 +1617,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
     }
     else if(!(conn->bits.no_body) &&
             conn->bits.chunk &&
-            (data->reqdata.proto.http->chunk.state != CHUNK_STOP)) {
+            (conn->chunk.state != CHUNK_STOP)) {
       /*
        * In chunked mode, return an error if the connection is closed prior to
        * the empty (terminiating) chunk is read.
diff --git a/lib/urldata.h b/lib/urldata.h
index b1de21ac5..4ed161a5f 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -296,7 +296,6 @@ struct HTTP {
 
   /* For FORM posting */
   struct Form form;
-  struct Curl_chunker chunk;
 
   struct back {
     curl_read_callback fread_func; /* backup storage for fread pointer */
@@ -818,6 +817,11 @@ struct connectdata {
      connection is used! */
   struct SessionHandle *data;
 
+  /* chunk is for HTTP chunked encoding, but is in the general connectdata
+     struct only because we can do just about any protocol through a HTTP proxy
+     and a HTTP proxy may in fact respond using chunked encoding */
+  struct Curl_chunker chunk;
+
   bool inuse; /* This is a marker for the connection cache logic. If this is
                  TRUE this handle is being used by an easy handle and cannot
                  be used by any other easy handle without careful