HTTP: implement Brotli content encoding

This uses the brotli external library (https://github.com/google/brotli).
Brotli becomes a feature: additional curl_version_info() bit and
structure fields are provided for it and CURLVERSION_NOW bumped.

Tests 314 and 315 check Brotli content unencoding with correct and
erroneous data.

Some tests are updated to accomodate with the now configuration dependent
parameters of the Accept-Encoding header.

3 files changed, 175 insertions, 3 deletions

diff --git a/lib/content_encoding.c b/lib/content_encoding.c
index 76a9e6866..6b3168573 100644
--- a/lib/content_encoding.c
+++ b/lib/content_encoding.c
@@ -37,14 +37,15 @@
 
 #ifndef CURL_DISABLE_HTTP
 
+#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
+
+
 #ifdef HAVE_LIBZ
 
 /* Comment this out if zlib is always going to be at least ver. 1.2.0.4
    (doing so will reduce code size slightly). */
 #define OLD_ZLIB_SUPPORT 1
 
-#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */
-
 #define GZIP_MAGIC_0 0x1f
 #define GZIP_MAGIC_1 0x8b
 
@@ -505,6 +506,136 @@ static const content_encoding gzip_encoding = {
 #endif /* HAVE_LIBZ */
 
 
+#ifdef HAVE_BROTLI
+
+/* Writer parameters. */
+typedef struct {
+  BrotliDecoderState *br;    /* State structure for brotli. */
+}  brotli_params;
+
+
+static CURLcode brotli_map_error(BrotliDecoderErrorCode be)
+{
+  switch(be) {
+  case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE:
+  case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE:
+  case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET:
+  case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME:
+  case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE:
+  case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE:
+  case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT:
+  case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1:
+  case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2:
+  case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM:
+  case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY:
+  case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS:
+  case BROTLI_DECODER_ERROR_FORMAT_PADDING_1:
+  case BROTLI_DECODER_ERROR_FORMAT_PADDING_2:
+  case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY:
+  case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET:
+  case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS:
+    return CURLE_BAD_CONTENT_ENCODING;
+  case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES:
+  case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS:
+  case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP:
+  case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1:
+  case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2:
+  case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES:
+    return CURLE_OUT_OF_MEMORY;
+  default:
+    break;
+  }
+  return CURLE_WRITE_ERROR;
+}
+
+static CURLcode brotli_init_writer(struct connectdata *conn,
+                                   contenc_writer *writer)
+{
+  brotli_params *bp = (brotli_params *) &writer->params;
+
+  (void) conn;
+
+  if(!writer->downstream)
+    return CURLE_WRITE_ERROR;
+
+  bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+  return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY;
+}
+
+static CURLcode brotli_unencode_write(struct connectdata *conn,
+                                      contenc_writer *writer,
+                                      const char *buf, size_t nbytes)
+{
+  brotli_params *bp = (brotli_params *) &writer->params;
+  const uint8_t *src = (const uint8_t *) buf;
+  char *decomp;
+  uint8_t *dst;
+  size_t dstleft;
+  CURLcode result = CURLE_OK;
+
+  if(!nbytes)
+    return CURLE_OK;
+  if(!bp->br)
+    return CURLE_WRITE_ERROR;  /* Stream already ended. */
+
+  decomp = malloc(DSIZ);
+  if(!decomp)
+    return CURLE_OUT_OF_MEMORY;
+
+  while(nbytes && result == CURLE_OK) {
+    BrotliDecoderResult r;
+
+    dst = (uint8_t *) decomp;
+    dstleft = DSIZ;
+    r = BrotliDecoderDecompressStream(bp->br,
+                                      &nbytes, &src, &dstleft, &dst, NULL);
+    result = Curl_unencode_write(conn, writer->downstream,
+                                 decomp, DSIZ - dstleft);
+    if(result)
+      break;
+    switch(r) {
+    case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+    case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+      break;
+    case BROTLI_DECODER_RESULT_SUCCESS:
+      BrotliDecoderDestroyInstance(bp->br);
+      bp->br = NULL;
+      if(nbytes)
+        result = CURLE_WRITE_ERROR;
+      break;
+    default:
+      result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br));
+      break;
+    }
+  }
+  free(decomp);
+  return result;
+}
+
+static void brotli_close_writer(struct connectdata *conn,
+                                contenc_writer *writer)
+{
+  brotli_params *bp = (brotli_params *) &writer->params;
+
+  (void) conn;
+
+  if(bp->br) {
+    BrotliDecoderDestroyInstance(bp->br);
+    bp->br = NULL;
+  }
+}
+
+static const content_encoding brotli_encoding = {
+  "br",
+  NULL,
+  brotli_init_writer,
+  brotli_unencode_write,
+  brotli_close_writer,
+  sizeof(brotli_params)
+};
+#endif
+
+
 /* Identity handler. */
 static CURLcode identity_init_writer(struct connectdata *conn,
                                      contenc_writer *writer)
@@ -544,6 +675,9 @@ static const content_encoding * const encodings[] = {
   &deflate_encoding,
   &gzip_encoding,
 #endif
+#ifdef HAVE_BROTLI
+  &brotli_encoding,
+#endif
   NULL
 };
 
diff --git a/lib/urldata.h b/lib/urldata.h
index cfdd8d028..7dfb26b6d 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -96,6 +96,10 @@
 #endif
 #endif
 
+#ifdef HAVE_BROTLI
+#include <brotli/decode.h>
+#endif
+
 #include <curl/curl.h>
 
 #include "http_chunks.h" /* for the structs and enum stuff */
diff --git a/lib/version.c b/lib/version.c
index ebd600635..66c761e34 100644
--- a/lib/version.c
+++ b/lib/version.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -74,6 +74,18 @@ void Curl_version_init(void)
   curl_version_info(CURLVERSION_NOW);
 }
 
+#ifdef HAVE_BROTLI
+static size_t brotli_version(char *buf, size_t bufsz)
+{
+  uint32_t brotli_version = BrotliDecoderVersion();
+  unsigned int major = brotli_version >> 24;
+  unsigned int minor = (brotli_version & 0x00FFFFFF) >> 12;
+  unsigned int patch = brotli_version & 0x00000FFF;
+
+  return snprintf(buf, bufsz, "%u.%u.%u", major, minor, patch);
+}
+#endif
+
 char *curl_version(void)
 {
   static bool initialized;
@@ -105,6 +117,14 @@ char *curl_version(void)
   left -= len;
   ptr += len;
 #endif
+#ifdef HAVE_BROTLI
+  len = snprintf(ptr, left, "%s", " brotli/");
+  left -= len;
+  ptr += len;
+  len = brotli_version(ptr, left);
+  left -= len;
+  ptr += len;
+#endif
 #ifdef USE_ARES
   /* this function is only present in c-ares, not in the original ares */
   len = snprintf(ptr, left, " c-ares/%s", ares_version(NULL));
@@ -327,6 +347,9 @@ static curl_version_info_data version_info = {
 #if defined(CURL_WITH_MULTI_SSL)
   | CURL_VERSION_MULTI_SSL
 #endif
+#if defined(HAVE_BROTLI)
+  | CURL_VERSION_BROTLI
+#endif
   ,
   NULL, /* ssl_version */
   0,    /* ssl_version_num, this is kept at zero */
@@ -337,6 +360,8 @@ static curl_version_info_data version_info = {
   NULL, /* libidn version */
   0,    /* iconv version */
   NULL, /* ssh lib version */
+  0,    /* brotli_ver_num */
+  NULL, /* brotli version */
 };
 
 curl_version_info_data *curl_version_info(CURLversion stamp)
@@ -348,6 +373,9 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
 #ifdef USE_SSL
   static char ssl_buffer[80];
 #endif
+#ifdef HAVE_BROTLI
+  static char brotli_buffer[80];
+#endif
 
   if(initialized)
     return &version_info;
@@ -396,6 +424,12 @@ curl_version_info_data *curl_version_info(CURLversion stamp)
   version_info.libssh_version = ssh_buffer;
 #endif
 
+#ifdef HAVE_BROTLI
+  version_info.brotli_ver_num = BrotliDecoderVersion();
+  brotli_version(brotli_buffer, sizeof brotli_buffer);
+  version_info.brotli_version = brotli_buffer;
+#endif
+
   (void)stamp; /* avoid compiler warnings, we don't use this */
 
   initialized = true;