- Shmulik Regev fixed so that the final CRLF of HTTP response headers are sent

  to the debug callback.

- Shmulik Regev added CURLOPT_HTTP_CONTENT_DECODING and
  CURLOPT_HTTP_TRANSFER_DECODING that if set to zero will disable libcurl's
  internal decoding of content or transfer encoded content. This may be
  preferable in cases where you use libcurl for proxy purposes or similar. The
  command line tool got a --raw option to disable both at once.

10 files changed, 86 insertions, 12 deletions

diff --git a/CHANGES b/CHANGES
index 1d395b7a7..2e19fd2c4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -7,6 +7,15 @@
                                   Changelog
 
 Daniel (12 February 2007)
+- Shmulik Regev fixed so that the final CRLF of HTTP response headers are sent
+  to the debug callback.
+
+- Shmulik Regev added CURLOPT_HTTP_CONTENT_DECODING and
+  CURLOPT_HTTP_TRANSFER_DECODING that if set to zero will disable libcurl's
+  internal decoding of content or transfer encoded content. This may be
+  preferable in cases where you use libcurl for proxy purposes or similar. The
+  command line tool got a --raw option to disable both at once.
+  
 - Jeff Pohlmeyer fixed a flaw in curl_multi_add_handle() when adding a handle
   that has an easy handle present in the "closure" list pending closure.
 
diff --git a/RELEASE-NOTES b/RELEASE-NOTES
index df3983db4..28d6623e3 100644
--- a/RELEASE-NOTES
+++ b/RELEASE-NOTES
@@ -2,8 +2,8 @@ Curl and libcurl 7.16.2
 
  Public curl release number:               98
  Releases counted from the very beginning: 125
- Available command line options:           115
- Available curl_easy_setopt() options:     137
+ Available command line options:           116
+ Available curl_easy_setopt() options:     141
  Number of public functions in libcurl:    54
  Amount of public web site mirrors:        39
  Number of known libcurl bindings:         35
@@ -12,6 +12,8 @@ Curl and libcurl 7.16.2
 This release includes the following changes:
  
  o Added CURLOPT_TIMEOUT_MS and CURLOPT_CONNECTTIMEOUT_MS
+ o Added CURLOPT_HTTP_CONTENT_DECODING, CURLOPT_HTTP_TRANSFER_DECODING and
+   --raw
 
 This release includes the following bugfixes:
 
@@ -34,6 +36,6 @@ New curl mirrors:
 This release would not have looked like this without help, code, reports and
 advice from friends like these:
 
- Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer
+ Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer, Shmulik Regev
 
         Thanks! (and sorry if I forgot to mention someone)
diff --git a/docs/curl.1 b/docs/curl.1
index 021f9505f..cd69f08ff 100644
--- a/docs/curl.1
+++ b/docs/curl.1
@@ -21,7 +21,7 @@
 .\" * $Id$
 .\" **************************************************************************
 .\"
-.TH curl 1 "3 Nov 2006" "Curl 7.16.1" "Curl Manual"
+.TH curl 1 "12 Feb 2007" "Curl 7.16.2" "Curl Manual"
 .SH NAME
 curl \- transfer a URL
 .SH SYNOPSIS
@@ -901,6 +901,11 @@ FTP range downloads only support the simple syntax 'start-stop' (optionally
 with one of the numbers omitted). It depends on the non-RFC command SIZE.
 
 If this option is used several times, the last one will be used.
+.IP "--raw"
+When used, it disables all internal HTTP decoding of content or transfer
+encodings and instead makes them passed on unaltered, raw. (Added in 7.16.2)
+
+If this option is used several times, each occurrence toggles this on/off.
 .IP "-R/--remote-time"
 When used, this will make libcurl attempt to figure out the timestamp of the
 remote file, and if that is available make the local file get that same
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 23d012de9..0c55e1400 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -21,7 +21,7 @@
 .\" * $Id$
 .\" **************************************************************************
 .\"
-.TH curl_easy_setopt 3 "5 Feb 2007" "libcurl 7.16.2" "libcurl Manual"
+.TH curl_easy_setopt 3 "12 Feb 2007" "libcurl 7.16.2" "libcurl Manual"
 .SH NAME
 curl_easy_setopt \- set options for a curl easy handle
 .SH SYNOPSIS
@@ -815,6 +815,16 @@ servers) which will report incorrect content length for files over 2
 gigabytes. If this option is used, curl will not be able to accurately report
 progress, and will simply stop the download when the server ends the
 connection. (added in 7.14.1)
+.IP CURLOPT_HTTP_CONTENT_DECODING
+Pass a long to tell libcurl how to act on content decoding. If set to zero,
+content decoding will be disabled. If set to 1 it is enabled. Note however
+that libcurl has no default content decoding but requires you to use
+\fICURLOPT_ENCODING\fP for that. (added in 7.16.2)
+.IP CURLOPT_HTTP_TRANSFER_DECODING
+Pass a long to tell libcurl how to act on transfer decoding. If set to zero,
+transfer decoding will be disabled, if set to 1 it is enabled
+(default). libcurl does chunked transfer decoding by default unless this
+option is set to zero. (added in 7.16.2)
 .RE
 .SH FTP OPTIONS
 .IP CURLOPT_FTPPORT
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 2c79f7dac..e911fca5a 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -1058,6 +1058,11 @@ typedef enum {
   CINIT(TIMEOUT_MS, LONG, 155),
   CINIT(CONNECTTIMEOUT_MS, LONG, 156),
 
+  /* set to zero to disable the libcurl's decoding and thus pass the raw body
+     data to the appliction even when it is encoded/compressed */
+  CINIT(HTTP_TRANSFER_DECODING, LONG, 157),
+  CINIT(HTTP_CONTENT_DECODING, LONG, 158),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 
diff --git a/lib/http_chunks.c b/lib/http_chunks.c
index 1b03a5569..36bee789c 100644
--- a/lib/http_chunks.c
+++ b/lib/http_chunks.c
@@ -116,6 +116,12 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 
   *wrote = 0; /* nothing's written yet */
 
+  /* the original data is written to the client, but we go on with the
+     chunk read process, to properly calculate the content length*/
+  if ( data->set.http_te_skip )
+    Curl_client_write(conn, CLIENTWRITE_BODY, datap,datalen);
+
+
   while(length) {
     switch(ch->state) {
     case CHUNK_HEX:
@@ -206,12 +212,17 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 
       /* Write the data portion available */
 #ifdef HAVE_LIBZ
-      switch (data->reqdata.keep.content_encoding) {
+      switch (conn->data->set.http_ce_skip?
+              IDENTITY : data->reqdata.keep.content_encoding) {
         case IDENTITY:
 #endif
-          if(!k->ignorebody)
-            result = Curl_client_write(conn, CLIENTWRITE_BODY, datap,
-                                       piece);
+          if(!k->ignorebody) {
+            if ( !data->set.http_te_skip )
+              result = Curl_client_write(conn, CLIENTWRITE_BODY, datap,
+                                         piece);
+            else
+              result = CURLE_OK;
+          }
 #ifdef HAVE_LIBZ
           break;
 
@@ -334,6 +345,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
             return(CHUNKE_BAD_CHUNK);
           }
 #endif /* CURL_DOES_CONVERSIONS */
+          if ( !data->set.http_te_skip )
           Curl_client_write(conn, CLIENTWRITE_HEADER,
                             conn->trailer, conn->trlPos);
         }
diff --git a/lib/transfer.c b/lib/transfer.c
index 828108204..cf4264cc8 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -689,6 +689,9 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                   k->keepon &= ~KEEP_READ;
                 }
 
+                if(data->set.verbose)
+                  Curl_debug(data, CURLINFO_HEADER_IN,
+                             k->str_start, headerlen, conn);
                 break;          /* exit header line loop */
               }
 
@@ -1286,7 +1289,8 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  Make sure that ALL_CONTENT_ENCODINGS contains all the
                  encodings handled here. */
 #ifdef HAVE_LIBZ
-              switch (k->content_encoding) {
+              switch (conn->data->set.http_ce_skip ?
+                      IDENTITY : k->content_encoding) {
               case IDENTITY:
 #endif
                 /* This is the default when the server sends no
diff --git a/lib/url.c b/lib/url.c
index 076f50ebd..148d7b2bb 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -1725,6 +1725,19 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
     data->set.ssh_private_key = va_arg(param, char *);
     break;
 
+  case CURLOPT_HTTP_TRANSFER_DECODING:
+    /*
+     * disable libcurl transfer encoding is used
+     */
+    data->set.http_te_skip = (bool)(0 == va_arg(param, long));
+    break;
+
+  case CURLOPT_HTTP_CONTENT_DECODING:
+    /*
+     * raw data passed to the application when content encoding is used
+     */
+    data->set.http_ce_skip = (bool)(0 == va_arg(param, long));
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_FAILED_INIT; /* correct this */
diff --git a/lib/urldata.h b/lib/urldata.h
index acc9d1ba6..3ba7fcacd 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1292,6 +1292,10 @@ struct UserDefined {
                              authentication */
   char *ssh_private_key;  /* the path to the private key file for
                              authentication */
+  bool http_te_skip;     /* pass the raw body data to the user, even when
+                            transfer-encoded (chunked, compressed) */
+  bool http_ce_skip;     /* pass the raw body data to the user, even when
+                            content-encoded (chunked, compressed) */
 };
 
 struct Names {
diff --git a/src/main.c b/src/main.c
index e35050025..d0b885151 100644
--- a/src/main.c
+++ b/src/main.c
@@ -20,7 +20,6 @@
  *
  * $Id$
  ***************************************************************************/
-
 #include "setup.h"
 
 #include <stdio.h>
@@ -470,7 +469,7 @@ struct Configurable {
   bool disable_sessionid;
 
   char *libcurl; /* output libcurl code to this file name */
-
+  bool raw;
   struct OutStruct *outs;
 };
 
@@ -683,6 +682,7 @@ static void help(void)
     " -Q/--quote <cmd>   Send command(s) to server before file transfer (F)",
     " -r/--range <range> Retrieve a byte range from a HTTP/1.1 or FTP server",
     "    --random-file <file> File for reading random data from (SSL)",
+    "    --raw           Pass HTTP \"raw\", without any transfer decoding (H)",
     " -R/--remote-time   Set the remote file's time on the local output",
     "    --retry <num>   Retry request <num> times if transient problems occur",
     "    --retry-delay <seconds> When retrying, wait this many seconds between each",
@@ -1473,6 +1473,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
     {"$x", "ftp-ssl-control", FALSE},
     {"$y", "ftp-ssl-ccc", FALSE},
     {"$z", "libcurl",    TRUE},
+    {"$#", "raw",        FALSE},
 
     {"0", "http1.0",     FALSE},
     {"1", "tlsv1",       FALSE},
@@ -1903,6 +1904,9 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
       case 'z': /* --libcurl */
         GetStr(&config->libcurl, nextarg);
         break;
+      case '#': /* --raw */
+        config->raw ^= TRUE;
+        break;
       }
       break;
     case '#': /* --progress-bar */
@@ -4253,6 +4257,12 @@ operate(struct Configurable *config, int argc, char *argv[])
         my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE,
                   !config->disable_sessionid);
 
+        /* curl 7.16.2 */
+        if(config->raw) {
+          my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE);
+          my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE);
+        }
+
         retry_numretries = config->req_retry;
 
         retrystart = curlx_tvnow();