James Gallagher's Content-Encoding work

19 files changed, 433 insertions, 27 deletions

diff --git a/CHANGES b/CHANGES
index 19ff00274..3a44c92c0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,13 @@
 
                                   Changelog
 
+Daniel (2 Sep 2002)
+- James Gallagher added Content-Encoding support to libcurl so now curl and
+  libcurl-using apps can request compressed contents using the 'deflate'
+  method. See the special file lib/README.encoding for details.
+
+  curl --compressed is now used to request compressed contents.
+
 Daniel (30 Aug 2002)
 - Applied an anonymous SOCKS5-proxy patch. Not properly working in all
   situations though, as all getaddrinfo()-using libcurls will fail on this.
diff --git a/Makefile.am b/Makefile.am
index 298c31971..164f7cc51 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -4,8 +4,7 @@
 
 AUTOMAKE_OPTIONS = foreign
 
-EXTRA_DIST =						\
-	CHANGES LEGAL maketgz MITX.txt MPL-1.1.txt	\
+EXTRA_DIST = CHANGES LEGAL maketgz MITX.txt MPL-1.1.txt UPGRADE		\
 	reconf Makefile.dist curl-config.in build_vms.com curl-mode.el
 
 bin_SCRIPTS = curl-config
diff --git a/UPGRADE b/UPGRADE
new file mode 100644
index 000000000..38d9679c5
--- /dev/null
+++ b/UPGRADE
@@ -0,0 +1,34 @@
+Upgrading to curl/libcurl 7.10 from any previous version
+========================================================
+
+libcurl 7.10 performs peer SSL certificate verification by default. This is
+done by installing a default CA cert bundle on 'make install' (or similar),
+that is used by default on operations against SSL servers.
+
+Alas, if you use communicate with HTTPS servers using certifcates that are
+signed by CAs present in the bundle, you will not notice and changed behavior
+and you will seeminglessly get a higher security level on your SSL connections
+since you will make sure that the remote server really is who it claims to be.
+
+If the remote server uses a self-signed certificate, or if you don't install
+curl's CA cert bundle or if it uses a certificate signed by a CA that isn't
+included in the bundle, then you need to do one of the following:
+
+ 1. Tell libcurl to *not* verify the peer. With libcurl you disable with with
+    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE);
+
+    With the curl command tool, you disable this with -k/--insecure.
+
+ 2. Get a CA certificate that can verify the remote server and use the proper
+    option to point out this CA cert for verification when connecting. For
+    libcurl hackers: curl_easy_setopt(curl, CURLOPT_CAPATH, capath);
+
+    With the curl command tool: --cacert [file]
+
+This upgrade procedure has been deemed The Right Thing even though it adds
+this extra trouble for some users, since it adds security to a majority of the
+SSL connections that previously weren't really secure.
+
+It turned out many people were using previous versions of curl/libcurl without
+realizing the need for the CA cert options to get truly secure SSL
+connections.
diff --git a/acconfig.h b/acconfig.h
index d84b28a4d..cceaa577c 100644
--- a/acconfig.h
+++ b/acconfig.h
@@ -85,3 +85,6 @@
 
 /* Define to disable TELNET */
 #undef CURL_DISABLE_TELNET
+
+/* Define if you have zlib present */
+#undef HAVE_LIBZ
diff --git a/configure.in b/configure.in
index ba45c060c..f4702b768 100644
--- a/configure.in
+++ b/configure.in
@@ -522,20 +522,21 @@ dnl NOTE:  We *always* look for ZLIB headers & libraries, all this option
 dnl        does is change where we look (by adjusting LIBS and CPPFLAGS.)
 dnl
 
-dnl AC_MSG_CHECKING(where to look for ZLIB)
-dnl if test X"$OPT_ZLIB" = Xno
-dnl then
-dnl 	AC_MSG_RESULT([defaults (or given in environment)])
-dnl else
-dnl	test X"$OPT_ZLIB" = Xyes && OPT_ZLIB=/usr/local
-dnl	LIBS="$LIBS -L$OPT_ZLIB/lib"
-dnl	CPPFLAGS="$CPPFLAGS -I$OPT_ZLIB/include"
-dnl	AC_MSG_RESULT([$OPT_ZLIB])
-dnl fi
-
-dnl z lib?
-dnl AC_CHECK_FUNC(gzread, , AC_CHECK_LIB(z, gzread))
+AC_MSG_CHECKING(where to look for ZLIB)
+if test X"$OPT_ZLIB" = Xno
+then
+	AC_MSG_RESULT([defaults (or given in environment)])
+else
+	test X"$OPT_ZLIB" = Xyes && OPT_ZLIB=/usr/local
+	LIBS="$LIBS -L$OPT_ZLIB/lib"
+	CPPFLAGS="$CPPFLAGS -I$OPT_ZLIB/include"
+	AC_MSG_RESULT([$OPT_ZLIB])
+fi
 
+dnl AC_CHECK_FUNC(gzread, , AC_CHECK_LIB(z, gzread))
+AC_CHECK_LIB(z, gzread, [AM_CONDITIONAL(CONTENT_ENCODING, true) 
+		         AC_DEFINE(HAVE_LIBZ)
+			 LIBS="$LIBS -lz"])
 
 dnl Default is to try the thread-safe versions of a few functions
 OPT_THREAD=on
@@ -606,7 +607,7 @@ AC_CHECK_HEADERS( \
 )
 
 dnl Check for libz header
-dnl AC_CHECK_HEADERS(zlib.h)
+AC_CHECK_HEADERS(zlib.h)
 
 dnl Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
diff --git a/docs/THANKS b/docs/THANKS
index f58fdffc4..81fd21f9b 100644
--- a/docs/THANKS
+++ b/docs/THANKS
@@ -81,3 +81,4 @@ that have contributed with non-trivial parts:
  - Götz Babin-Ebell <babin­ebell@trustcenter.de>
  - Andreas Damm <andreas-sourceforge@radab.org>
  - Jacky Lam <sylam@emsoftltd.com>
+ - James Gallagher <jgallagher@gso.uri.edu>
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 1b20a674e..e547136ac 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -200,6 +200,7 @@ typedef enum {
   CURLE_SSL_CERTPROBLEM,         /* 58 - problem with the local certificate */
   CURLE_SSL_CIPHER,              /* 59 - couldn't use specified cipher */
   CURLE_SSL_CACERT,              /* 60 - problem with the CA cert (path?) */
+  CURLE_BAD_CONTENT_ENCODING,    /* 61 - Unrecognized transfer encoding */
 
   CURL_LAST /* never use! */
 } CURLcode;
@@ -585,6 +586,11 @@ typedef enum {
      CURLPROXY_SOCKS4 and CURLPROXY_SOCKS5. */
   CINIT(PROXYTYPE, LONG, 101),
 
+  /* Set the Accept-Encoding string. Use this to tell a server you would like
+     the response to be compressed. */
+  CINIT(ENCODING, OBJECTPOINT, 102),
+ 
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 12a582d92..cf0258f2c 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -7,7 +7,7 @@ AUTOMAKE_OPTIONS = foreign nostdinc
 EXTRA_DIST = getdate.y Makefile.b32 Makefile.b32.resp Makefile.m32	    \
        Makefile.vc6 Makefile.riscos libcurl.def dllinit.c curllib.dsp	    \
        curllib.dsw config-vms.h config-win32.h config-riscos.h config-mac.h \
-       config.h.in ca-bundle.crt
+       config.h.in ca-bundle.crt README.encoding
 
 lib_LTLIBRARIES = libcurl.la
 
@@ -45,6 +45,8 @@ libcurl_la_LDFLAGS = -version-info 2:2:0
 # set age to 0.
 #
 
+# I could not get conditional compilation using LIBADD or LDADD and
+# EXTRA_..._SOURCES to work. 08/28/02 jhrg
 libcurl_la_SOURCES = \
 arpa_telnet.h  file.c         getpass.h      netrc.h        timeval.c \
 base64.c       file.h         hostip.c       progress.c     timeval.h \
@@ -60,7 +62,7 @@ escape.h       getpass.c      netrc.c        telnet.h       \
 getinfo.c getinfo.h transfer.c strequal.c strequal.h easy.c \
 security.h security.c krb4.c krb4.h memdebug.c memdebug.h inet_ntoa_r.h \
 http_chunks.c http_chunks.h strtok.c strtok.h connect.c connect.h \
-llist.c llist.h hash.c hash.h multi.c share.c share.h
+llist.c llist.h hash.c hash.h multi.c content_encoding.c
 
 noinst_HEADERS = setup.h transfer.h
 
diff --git a/lib/README.encoding b/lib/README.encoding
new file mode 100644
index 000000000..ef5c8036f
--- /dev/null
+++ b/lib/README.encoding
@@ -0,0 +1,53 @@
+
+		    Content Encoding Support for libcurl
+
+* About content encodings: 
+
+HTTP/1.1 [RFC 2616] specifies that a client may request that a server encode
+its response. This is usually used to compress a response using one of a set
+of commonly available compression techniques. These schemes are `deflate'
+(the zlib algorithm), `gzip' and `compress' [sec 3.5, RFC 2616]. A client
+requests that the sever perform an encoding by including an Accept-Encoding
+header in the request document. The value of the header should be one of the
+recognized tokens `deflate', ... (there's a way to register new
+schemes/tokens, see sec 3.5 of the spec). A server MAY honor the client's
+encoding request. When a response is encoded, the server includes a
+Content-Encoding header in the response. The value of the Content-Encoding
+header indicates which scheme was used to encode the data.
+
+A client may tell a server that it can understand several different encoding
+schemes. In this case the server may choose any one of those and use it to
+encode the response (indicating which one using the Content-Encoding header).
+It's also possible for a client to attach priorities to different schemes so
+that the server knows which it prefers. See sec 14.3 of RFC 2616 for more
+information on the Accept-Encoding header.
+
+* Current support for content encoding:
+
+I added support for the 'deflate' content encoding to both libcurl and curl.
+Both regular and chunked transfers should work although I've tested only the
+former. The library zlib is required for this feature. Places where I
+modified the source code are commented and typically include my initials and
+the date (e.g., 08/29/02 jhrg).
+
+* The libcurl interface:
+
+To cause libcurl to request a content encoding use: 
+
+    curl_easy_setopt(curl, CURLOPT_ENCODING, <string>) 
+
+where <string> is the intended value of the Accept-Encoding header.
+
+Currently, libcurl only understands how to process responses that use the
+`deflate' Content-Encoding, so the only value for CURLOPT_ENCODING that will
+work (besides "identity," which does nothing) is "deflate." If a response is
+encoded using either the `gzip' or `compress' methods, libcurl will return an
+error indicating that the response could not be decoded. If <string> is null
+or empty no Accept-Encoding header is generated.
+
+* The curl interface:
+
+Use the --compressed option with curl to cause it to ask servers to compress
+responses using deflate. 
+
+James Gallagher <jgallagher@gso.uri.edu>
diff --git a/lib/content_encoding.c b/lib/content_encoding.c
new file mode 100644
index 000000000..51b59c584
--- /dev/null
+++ b/lib/content_encoding.c
@@ -0,0 +1,122 @@
+/*****************************************************************************
+ *                                  _   _ ____  _     
+ *  Project                     ___| | | |  _ \| |    
+ *                             / __| | | | |_) | |    
+ *                            | (__| |_| |  _ <| |___ 
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2002, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * In order to be useful for every potential user, curl and libcurl are
+ * dual-licensed under the MPL and the MIT/X-derivate licenses.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the MPL or the MIT/X-derivate
+ * licenses. You may pick one of these licenses.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * $Id$
+ *****************************************************************************/
+
+#include "setup.h"
+
+#ifdef HAVE_LIBZ
+
+#include "urldata.h"
+#include <curl/curl.h>
+#include <curl/types.h>
+#include "sendf.h"
+
+#define DSIZ 4096               /* buffer size for decompressed data */
+
+
+static CURLcode
+process_zlib_error(struct SessionHandle *data, z_stream *z)
+{
+  if (z->msg)
+    failf (data, "Error while processing content unencoding.\n%s",
+           z->msg);
+  else
+    failf (data, "Error while processing content unencoding.\n"
+           "Unknown failure within decompression software.");
+
+  return CURLE_BAD_CONTENT_ENCODING;
+}
+
+static CURLcode
+exit_zlib(z_stream *z, bool *zlib_init, CURLcode result)
+{
+  inflateEnd(z);
+  *zlib_init = 0;
+  return result;
+}
+
+CURLcode
+Curl_unencode_deflate_write(struct SessionHandle *data, 
+                            struct Curl_transfer_keeper *k,
+                            ssize_t nread)
+{
+  int status;                   /* zlib status */
+  int result;                   /* Curl_client_write status */
+  char decomp[DSIZ];            /* Put the decompressed data here. */
+  z_stream *z = &k->z;          /* zlib state structure */
+              
+  /* Initialize zlib? */
+  if (!k->zlib_init) {
+    z->zalloc = (alloc_func)Z_NULL;
+    z->zfree = (free_func)Z_NULL;
+    z->opaque = 0;              /* of dubious use 08/27/02 jhrg */
+    if (inflateInit(z) != Z_OK)
+      return process_zlib_error(data, z);
+    k->zlib_init = 1;
+  }
+
+  /* Set the compressed input when this fucntion is called */
+  z->next_in = (Bytef *)k->str;
+  z->avail_in = nread;
+
+  /* because the buffer size is fixed, iteratively decompress
+     and transfer to the client via client_write. */
+  for (;;) {
+    /* (re)set buffer for decompressed output for every iteration */
+    z->next_out = (Bytef *)&decomp[0];
+    z->avail_out = DSIZ;
+
+    status = inflate(z, Z_SYNC_FLUSH);
+    if (status == Z_OK || status == Z_STREAM_END) {
+      result = Curl_client_write(data, CLIENTWRITE_BODY, decomp, 
+                                 DSIZ - z->avail_out);
+      /* if !CURLE_OK, clean up, return */
+      if (result) {              
+        return exit_zlib(z, &k->zlib_init, result);
+      }
+
+      /* Done?; clean up, return */
+      if (status == Z_STREAM_END) {
+        if (inflateEnd(z) == Z_OK)
+          return exit_zlib(z, &k->zlib_init, result);
+        else
+          return exit_zlib(z, &k->zlib_init, process_zlib_error(data, z));
+      }
+
+      /* Done with these bytes, exit */
+      if (status == Z_OK && z->avail_in == 0 && z->avail_out > 0) 
+        return result;
+    }
+    else {                      /* Error; exit loop, handle below */
+      return exit_zlib(z, &k->zlib_init, process_zlib_error(data, z));
+    }
+  }
+}
+#endif /* HAVE_LIBZ */
+
+/*
+ * local variables:
+ * eval: (load-file "../curl-mode.el")
+ * end:
+ * vim600: fdm=marker
+ * vim: et sw=2 ts=2 sts=2 tw=78
+ */
diff --git a/lib/content_encoding.h b/lib/content_encoding.h
new file mode 100644
index 000000000..91ba7d110
--- /dev/null
+++ b/lib/content_encoding.h
@@ -0,0 +1,34 @@
+/*****************************************************************************
+ *                                  _   _ ____  _     
+ *  Project                     ___| | | |  _ \| |    
+ *                             / __| | | | |_) | |    
+ *                            | (__| |_| |  _ <| |___ 
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2002, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * In order to be useful for every potential user, curl and libcurl are
+ * dual-licensed under the MPL and the MIT/X-derivate licenses.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the MPL or the MIT/X-derivate
+ * licenses. You may pick one of these licenses.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ * $Id$
+ *****************************************************************************/
+
+CURLcode Curl_unencode_deflate_write(struct SessionHandle *data, 
+                                     struct Curl_transfer_keeper *k, 
+                                     ssize_t nread);
+          
+/*
+ * local variables:
+ * eval: (load-file "../curl-mode.el")
+ * end:
+ * vim600: fdm=marker
+ * vim: et sw=2 ts=2 sts=2 tw=78
+ */
diff --git a/lib/http.c b/lib/http.c
index 08a5536dc..8957968d8 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -707,6 +707,7 @@ CURLcode Curl_http(struct connectdata *conn)
                 "%s" /* host */
                 "%s" /* pragma */
                 "%s" /* accept */
+                "%s" /* accept-encoding */
                 "%s", /* referer */
 
                 data->set.customrequest?data->set.customrequest:
@@ -727,6 +728,8 @@ CURLcode Curl_http(struct connectdata *conn)
                 (conn->allocptr.host?conn->allocptr.host:""), /* Host: host */
                 http->p_pragma?http->p_pragma:"",
                 http->p_accept?http->p_accept:"",
+                (data->set.encoding && *data->set.encoding && conn->allocptr.accept_encoding)?
+                conn->allocptr.accept_encoding:"", /* 08/28/02 jhrg */
                 (data->change.referer && conn->allocptr.ref)?conn->allocptr.ref:"" /* Referer: <data> <CRLF> */
                 );
 
diff --git a/lib/http_chunks.c b/lib/http_chunks.c
index 939e86a91..784d231fe 100644
--- a/lib/http_chunks.c
+++ b/lib/http_chunks.c
@@ -33,6 +33,8 @@
 #include "urldata.h" /* it includes http_chunks.h */
 #include "sendf.h"   /* for the client write stuff */
 
+#include "content_encoding.h"   /* 08/29/02 jhrg */
+
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
 
@@ -172,7 +174,32 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
       piece = (ch->datasize >= length)?length:ch->datasize;
 
       /* Write the data portion available */
-      result = Curl_client_write(conn->data, CLIENTWRITE_BODY, datap, piece);
+      /* Added content-encoding here; untested but almost identical to the
+         tested code in transfer.c. 08/29/02 jhrg */
+#ifdef HAVE_LIBZ
+      switch (conn->keep.content_encoding) {
+        case IDENTITY:
+#endif
+          result = Curl_client_write(conn->data, CLIENTWRITE_BODY, datap,
+                                     piece);
+#ifdef HAVE_LIBZ
+          break;
+
+        case DEFLATE: 
+          result = Curl_unencode_deflate_write(conn->data, &conn->keep, piece);
+          break;
+
+        case GZIP:
+        case COMPRESS:
+        default:
+          failf (conn->data,
+                 "Unrecognized content encoding type. "
+                 "libcurl understands `identity' and `deflate' "
+                 "content encodings.");
+          return CHUNKE_BAD_ENCODING;
+      }
+#endif
+
       if(result)
         return CHUNKE_WRITE_ERROR;
       *wrote += piece;
diff --git a/lib/http_chunks.h b/lib/http_chunks.h
index 48bdbd37b..482612d25 100644
--- a/lib/http_chunks.h
+++ b/lib/http_chunks.h
@@ -73,6 +73,7 @@ typedef enum {
   CHUNKE_BAD_CHUNK,
   CHUNKE_WRITE_ERROR,
   CHUNKE_STATE_ERROR,
+  CHUNKE_BAD_ENCODING,
   CHUNKE_LAST
 } CHUNKcode;
 
diff --git a/lib/transfer.c b/lib/transfer.c
index 20088b17d..2567ca91b 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -82,6 +82,8 @@
 #include <curl/types.h>
 #include "netrc.h"
 
+#include "content_encoding.h"   /* content encoding support. 08/27/02 jhrg */
+
 #include "hostip.h"
 #include "transfer.h"
 #include "sendf.h"
@@ -368,7 +370,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                * we got: "417 Expectation Failed" this means:
                * we have made a HTTP call and our Expect Header
                * seems to cause a problem => abort the write operations
-               * (or prevent them from starting
+               * (or prevent them from starting).
                */
               k->write_after_100_header = FALSE;
               k->keepon &= ~KEEP_WRITE;
@@ -575,6 +577,34 @@ CURLcode Curl_readwrite(struct connectdata *conn,
             /* init our chunky engine */
             Curl_httpchunk_init(conn);
           }
+          else if (strnequal("Content-Encoding:", k->p, 17) &&
+                   data->set.encoding) {
+            /*
+             * Process Content-Encoding. Look for the values: identity, gzip,
+             * defalte, compress, x-gzip and x-compress. x-gzip and
+             * x-compress are the same as gzip and compress. (Sec 3.5 RFC
+             * 2616). zlib cannot handle compress, and gzip is not currently
+             * implemented. However, errors are handled further down when the
+             * response body is processed 08/27/02 jhrg */
+            char *start;
+
+            /* Find the first non-space letter */
+            for(start=k->p+18;
+                *start && isspace((int)*start);
+                start++);
+
+            /* Record the content-encoding for later use. 08/27/02 jhrg */
+            if (strnequal("identity", start, 8))
+              k->content_encoding = IDENTITY;
+            else if (strnequal("deflate", start, 7))
+              k->content_encoding = DEFLATE;
+            else if (strnequal("gzip", start, 4) 
+                     || strnequal("x-gzip", start, 6))
+              k->content_encoding = GZIP;
+            else if (strnequal("compress", start, 8) 
+                     || strnequal("x-compress", start, 10))
+              k->content_encoding = COMPRESS;
+          }
           else if (strnequal("Content-Range:", k->p, 14)) {
             if (sscanf (k->p+14, " bytes %d-", &k->offset) ||
                 sscanf (k->p+14, " bytes: %d-", &k->offset)) {
@@ -737,6 +767,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
            * the name says read, this function both reads and writes away
            * the data. The returned 'nread' holds the number of actual
            * data it wrote to the client.  */
+          /* Handle chunking here? 08/27/02 jhrg */
           CHUNKcode res =
             Curl_httpchunk_read(conn, k->str, nread, &nread);
 
@@ -776,8 +807,39 @@ CURLcode Curl_readwrite(struct connectdata *conn,
             
         if(!conn->bits.chunk && nread) {
           /* If this is chunky transfer, it was already written */
-          result = Curl_client_write(data, CLIENTWRITE_BODY, k->str,
-                                     nread);
+
+          /* This switch handles various content encodings. If there's an
+             error here, be sure to check over the almost identical code in
+             http_chunk.c. 08/29/02 jhrg */
+#ifdef HAVE_LIBZ
+          switch (k->content_encoding) {
+            case IDENTITY:
+#endif
+              /* This is the default when the server sends no
+                 Content-Encoding header. See Curl_readwrite_init; the
+                 memset() call initializes k->content_encoding to zero.
+                 08/28/02 jhrg */
+              result = Curl_client_write(data, CLIENTWRITE_BODY, k->str, 
+                                         nread);
+#ifdef HAVE_LIBZ
+              break;
+
+            case DEFLATE: 
+              /* Assume CLIENTWRITE_BODY; headers are not encoded. */
+              result = Curl_unencode_deflate_write(data, k, nread);
+              break;
+
+            case GZIP:          /* FIXME 08/27/02 jhrg */
+            case COMPRESS:
+            default:
+              failf (data, "Unrecognized content encoding type. "
+                     "libcurl understands `identity' and `deflate' "
+                     "content encodings.");
+              result = CURLE_BAD_CONTENT_ENCODING;
+              break;
+          }
+#endif
+
           if(result)
             return result;
         }
@@ -954,6 +1016,8 @@ CURLcode Curl_readwrite_init(struct connectdata *conn)
   struct SessionHandle *data = conn->data;
   struct Curl_transfer_keeper *k = &conn->keep;
 
+  /* NB: the content encoding software depends on this initialization of
+     Curl_transfer_keeper. 08/28/02 jhrg */
   memset(k, 0, sizeof(struct Curl_transfer_keeper));
 
   k->start = Curl_tvnow(); /* start time */
diff --git a/lib/url.c b/lib/url.c
index f8d647491..26c5cb85f 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -303,7 +303,7 @@ CURLcode Curl_open(struct SessionHandle **curl)
   data->set.ssl.verifyhost = 2;
 #ifdef CURL_CA_BUNDLE
   /* This is our prefered CA cert bundle since install time */
-  data->set.ssl.CAfile = CURL_CA_BUNDLE;
+  data->set.ssl.CAfile = (char *)CURL_CA_BUNDLE;
 #endif
 
 
@@ -781,6 +781,13 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, ...)
      */
     data->set.useragent = va_arg(param, char *);
     break;
+  case CURLOPT_ENCODING:
+    /*
+     * String to use at the value of Accept-Encoding header. 08/28/02 jhrg
+     */
+    data->set.encoding = va_arg(param, char *);
+    break;
+
   case CURLOPT_USERPWD:
     /*
      * user:password to use in the operation
@@ -1127,6 +1134,8 @@ CURLcode Curl_disconnect(struct connectdata *conn)
     free(conn->allocptr.uagent);
   if(conn->allocptr.userpwd)
     free(conn->allocptr.userpwd);
+  if(conn->allocptr.accept_encoding)
+    free(conn->allocptr.accept_encoding);
   if(conn->allocptr.rangeline)
     free(conn->allocptr.rangeline);
   if(conn->allocptr.ref)
@@ -2715,6 +2724,12 @@ static CURLcode CreateConnection(struct SessionHandle *data,
     }
   }
 
+    if(data->set.encoding) {
+      if(conn->allocptr.accept_encoding)
+        free(conn->allocptr.accept_encoding);
+      conn->allocptr.accept_encoding =
+        aprintf("Accept-Encoding: %s\015\012", data->set.encoding);
+    }
   conn->bytecount = 0;
   conn->headerbytecount = 0;
   
diff --git a/lib/urldata.h b/lib/urldata.h
index 2f183711e..74a7c492e 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -82,6 +82,10 @@
 
 #include "http_chunks.h" /* for the structs and enum stuff */
 
+#ifdef HAVE_ZLIB_H
+#include <zlib.h> 		/* for content-encoding 08/28/02 jhrg */
+#endif
+
 /* Download buffer size, keep it fairly big for speed reasons */
 #define BUFSIZE CURL_MAX_WRITE_SIZE
 
@@ -241,6 +245,20 @@ struct Curl_transfer_keeper {
                                    we received a 100-continue/timeout
                                    or directly */
 
+  /* for content-encoding 08/28/02 jhrg */
+  int content_encoding;  	/* What content encoding. sec 3.5, RFC2616. */
+
+#define IDENTITY 0		/* No encoding */
+#define DEFLATE 1		/* zlib delfate [RFC 1950 & 1951] */
+#define GZIP 2			/* gzip algorithm [RFC 1952] */
+#define COMPRESS 3		/* Not handled, added for completeness */
+
+#ifdef HAVE_LIBZ
+  bool zlib_init;		/* True if zlib already initialized;
+				   undefined if Content-Encdoing header. */
+  z_stream z;			/* State structure for zlib. */
+#endif
+
   /* for the low speed checks: */
   time_t timeofdoc;
   long bodywrites;
@@ -365,6 +383,7 @@ struct connectdata {
   struct dynamically_allocated_data {
     char *proxyuserpwd; /* free later if not NULL! */
     char *uagent; /* free later if not NULL! */
+    char *accept_encoding; /* free later if not NULL! 08/28/02 jhrg */
     char *userpwd; /* free later if not NULL! */
     char *rangeline; /* free later if not NULL! */
     char *ref; /* free later if not NULL! */
@@ -424,6 +443,8 @@ struct connectdata {
                    supposed to be called, after ->curl_do() */
 };
 
+/* The end of connectdata. 08/27/02 jhrg */
+
 /*
  * Struct to keep statistical and informational data.
  */
@@ -593,6 +614,7 @@ struct UserDefined {
   bool free_referer; /* set TRUE if 'referer' points to a string we
                         allocated */
   char *useragent;   /* User-Agent string */
+  char *encoding;    /* Accept-Encoding string 08/28/02 jhrg */
   char *postfields;  /* if POST, set the fields' values here */
   size_t postfieldsize; /* if POST, this might have a size to use instead of
                            strlen(), and then the data *may* be binary (contain
diff --git a/src/Makefile.am b/src/Makefile.am
index 82433a659..27c4a0bcd 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -18,7 +18,7 @@ curl_SOURCES = main.c hugehelp.c urlglob.c writeout.c setup.h \
 	config-win32.h config-mac.h config-vms.h config-riscos.h \
 	urlglob.h version.h writeout.h writeenv.c writeenv.h
 
-curl_LDADD = ../lib/libcurl.la
+curl_LDADD = ../lib/libcurl.la -lz
 curl_DEPENDENCIES = ../lib/libcurl.la
 BUILT_SOURCES = hugehelp.c
 CLEANFILES = hugehelp.c
diff --git a/src/main.c b/src/main.c
index fa2b43f38..9c8172477 100644
--- a/src/main.c
+++ b/src/main.c
@@ -352,8 +352,10 @@ static void help(void)
        "    --cacert <file> CA certifciate to verify peer against (SSL)\n"
        "    --capath <directory> CA directory (made using c_rehash) to verify\n"
        "                    peer against (SSL, NOT Windows)\n"
-       "    --ciphers <list> What SSL ciphers to use (SSL)");
+       "    --ciphers <list> What SSL ciphers to use (SSL)\n"
+       "    --compressed    Request a compressed response (using deflate).");
   puts("    --connect-timeout <seconds> Maximum time allowed for connection\n"
+       "    --crlf          Convert LF to CRLF in upload. Useful for MVS (OS/390)\n"
        " -f/--fail          Fail silently (no output at all) on errors (H)\n"
        " -F/--form <name=content> Specify HTTP POST data (H)\n"
        " -g/--globoff       Disable URL sequences and ranges using {} and []\n"
@@ -408,8 +410,7 @@ static void help(void)
        " -1/--tlsv1         Force usage of TLSv1 (H)\n"
        " -2/--sslv2         Force usage of SSLv2 (H)\n"
        " -3/--sslv3         Force usage of SSLv3 (H)");
-  puts(" -#/--progress-bar  Display transfer progress as a progress bar\n"
-       "    --crlf          Convert LF to CRLF in upload. Useful for MVS (OS/390)");
+  puts(" -#/--progress-bar  Display transfer progress as a progress bar");
 }
 
 struct LongShort {
@@ -427,6 +428,7 @@ struct Configurable {
   char *cookiejar;  /* write to this file */
   char *cookiefile; /* read from this file */
   bool cookiesession; /* new session? */
+  bool encoding;    /* Accept-Encoding please */
   bool use_resume;
   bool resume_from_current;
   bool disable_epsv;
@@ -999,6 +1001,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
     {"5g", "trace",      TRUE},
     {"5h", "trace-ascii", TRUE},
     {"5i", "limit-rate", TRUE},
+    {"5j", "compressed",  FALSE}, /* might take an arg someday */
     {"0", "http1.0",     FALSE},
     {"1", "tlsv1",       FALSE},
     {"2", "sslv2",       FALSE},
@@ -1213,6 +1216,11 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
           config->sendpersecond = value;
         }
         break;
+
+      case 'j': /* --compressed */
+ 	config->encoding ^= TRUE;
+ 	break;
+
       default: /* the URL! */
         {
           struct getout *url;
@@ -2806,6 +2814,10 @@ operate(struct Configurable *config, int argc, char *argv[])
       }
       curl_easy_setopt(curl, CURLOPT_VERBOSE, config->conf&CONF_VERBOSE);
 
+      /* new in curl 7.10 */
+      curl_easy_setopt(curl, CURLOPT_ENCODING, 
+                       (config->encoding) ? "deflate" : NULL);
+
       res = curl_easy_perform(curl);
         
       if((config->progressmode == CURL_PROGRESS_BAR) &&