From 11bf1796cd015373a996e6eb26212e2e1aadb066 Mon Sep 17 00:00:00 2001 From: Patrick Monnerat Date: Sun, 5 Nov 2017 15:28:16 +0100 Subject: HTTP: implement Brotli content encoding This uses the brotli external library (https://github.com/google/brotli). Brotli becomes a feature: additional curl_version_info() bit and structure fields are provided for it and CURLVERSION_NOW bumped. Tests 314 and 315 check Brotli content unencoding with correct and erroneous data. Some tests are updated to accomodate with the now configuration dependent parameters of the Accept-Encoding header. --- lib/content_encoding.c | 138 ++++++++++++++++++++++++++++++++++++++++++++++++- lib/urldata.h | 4 ++ lib/version.c | 36 ++++++++++++- 3 files changed, 175 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/content_encoding.c b/lib/content_encoding.c index 76a9e6866..6b3168573 100644 --- a/lib/content_encoding.c +++ b/lib/content_encoding.c @@ -37,14 +37,15 @@ #ifndef CURL_DISABLE_HTTP +#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */ + + #ifdef HAVE_LIBZ /* Comment this out if zlib is always going to be at least ver. 1.2.0.4 (doing so will reduce code size slightly). */ #define OLD_ZLIB_SUPPORT 1 -#define DSIZ CURL_MAX_WRITE_SIZE /* buffer size for decompressed data */ - #define GZIP_MAGIC_0 0x1f #define GZIP_MAGIC_1 0x8b @@ -505,6 +506,136 @@ static const content_encoding gzip_encoding = { #endif /* HAVE_LIBZ */ +#ifdef HAVE_BROTLI + +/* Writer parameters. */ +typedef struct { + BrotliDecoderState *br; /* State structure for brotli. */ +} brotli_params; + + +static CURLcode brotli_map_error(BrotliDecoderErrorCode be) +{ + switch(be) { + case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE: + case BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE: + case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET: + case BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME: + case BROTLI_DECODER_ERROR_FORMAT_CL_SPACE: + case BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE: + case BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT: + case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1: + case BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2: + case BROTLI_DECODER_ERROR_FORMAT_TRANSFORM: + case BROTLI_DECODER_ERROR_FORMAT_DICTIONARY: + case BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: + case BROTLI_DECODER_ERROR_FORMAT_PADDING_1: + case BROTLI_DECODER_ERROR_FORMAT_PADDING_2: + case BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY: + case BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET: + case BROTLI_DECODER_ERROR_INVALID_ARGUMENTS: + return CURLE_BAD_CONTENT_ENCODING; + case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES: + case BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS: + case BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP: + case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1: + case BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2: + case BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES: + return CURLE_OUT_OF_MEMORY; + default: + break; + } + return CURLE_WRITE_ERROR; +} + +static CURLcode brotli_init_writer(struct connectdata *conn, + contenc_writer *writer) +{ + brotli_params *bp = (brotli_params *) &writer->params; + + (void) conn; + + if(!writer->downstream) + return CURLE_WRITE_ERROR; + + bp->br = BrotliDecoderCreateInstance(NULL, NULL, NULL); + return bp->br? CURLE_OK: CURLE_OUT_OF_MEMORY; +} + +static CURLcode brotli_unencode_write(struct connectdata *conn, + contenc_writer *writer, + const char *buf, size_t nbytes) +{ + brotli_params *bp = (brotli_params *) &writer->params; + const uint8_t *src = (const uint8_t *) buf; + char *decomp; + uint8_t *dst; + size_t dstleft; + CURLcode result = CURLE_OK; + + if(!nbytes) + return CURLE_OK; + if(!bp->br) + return CURLE_WRITE_ERROR; /* Stream already ended. */ + + decomp = malloc(DSIZ); + if(!decomp) + return CURLE_OUT_OF_MEMORY; + + while(nbytes && result == CURLE_OK) { + BrotliDecoderResult r; + + dst = (uint8_t *) decomp; + dstleft = DSIZ; + r = BrotliDecoderDecompressStream(bp->br, + &nbytes, &src, &dstleft, &dst, NULL); + result = Curl_unencode_write(conn, writer->downstream, + decomp, DSIZ - dstleft); + if(result) + break; + switch(r) { + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + break; + case BROTLI_DECODER_RESULT_SUCCESS: + BrotliDecoderDestroyInstance(bp->br); + bp->br = NULL; + if(nbytes) + result = CURLE_WRITE_ERROR; + break; + default: + result = brotli_map_error(BrotliDecoderGetErrorCode(bp->br)); + break; + } + } + free(decomp); + return result; +} + +static void brotli_close_writer(struct connectdata *conn, + contenc_writer *writer) +{ + brotli_params *bp = (brotli_params *) &writer->params; + + (void) conn; + + if(bp->br) { + BrotliDecoderDestroyInstance(bp->br); + bp->br = NULL; + } +} + +static const content_encoding brotli_encoding = { + "br", + NULL, + brotli_init_writer, + brotli_unencode_write, + brotli_close_writer, + sizeof(brotli_params) +}; +#endif + + /* Identity handler. */ static CURLcode identity_init_writer(struct connectdata *conn, contenc_writer *writer) @@ -543,6 +674,9 @@ static const content_encoding * const encodings[] = { #ifdef HAVE_LIBZ &deflate_encoding, &gzip_encoding, +#endif +#ifdef HAVE_BROTLI + &brotli_encoding, #endif NULL }; diff --git a/lib/urldata.h b/lib/urldata.h index cfdd8d028..7dfb26b6d 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -96,6 +96,10 @@ #endif #endif +#ifdef HAVE_BROTLI +#include +#endif + #include #include "http_chunks.h" /* for the structs and enum stuff */ diff --git a/lib/version.c b/lib/version.c index ebd600635..66c761e34 100644 --- a/lib/version.c +++ b/lib/version.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2017, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -74,6 +74,18 @@ void Curl_version_init(void) curl_version_info(CURLVERSION_NOW); } +#ifdef HAVE_BROTLI +static size_t brotli_version(char *buf, size_t bufsz) +{ + uint32_t brotli_version = BrotliDecoderVersion(); + unsigned int major = brotli_version >> 24; + unsigned int minor = (brotli_version & 0x00FFFFFF) >> 12; + unsigned int patch = brotli_version & 0x00000FFF; + + return snprintf(buf, bufsz, "%u.%u.%u", major, minor, patch); +} +#endif + char *curl_version(void) { static bool initialized; @@ -105,6 +117,14 @@ char *curl_version(void) left -= len; ptr += len; #endif +#ifdef HAVE_BROTLI + len = snprintf(ptr, left, "%s", " brotli/"); + left -= len; + ptr += len; + len = brotli_version(ptr, left); + left -= len; + ptr += len; +#endif #ifdef USE_ARES /* this function is only present in c-ares, not in the original ares */ len = snprintf(ptr, left, " c-ares/%s", ares_version(NULL)); @@ -326,6 +346,9 @@ static curl_version_info_data version_info = { #endif #if defined(CURL_WITH_MULTI_SSL) | CURL_VERSION_MULTI_SSL +#endif +#if defined(HAVE_BROTLI) + | CURL_VERSION_BROTLI #endif , NULL, /* ssl_version */ @@ -337,6 +360,8 @@ static curl_version_info_data version_info = { NULL, /* libidn version */ 0, /* iconv version */ NULL, /* ssh lib version */ + 0, /* brotli_ver_num */ + NULL, /* brotli version */ }; curl_version_info_data *curl_version_info(CURLversion stamp) @@ -348,6 +373,9 @@ curl_version_info_data *curl_version_info(CURLversion stamp) #ifdef USE_SSL static char ssl_buffer[80]; #endif +#ifdef HAVE_BROTLI + static char brotli_buffer[80]; +#endif if(initialized) return &version_info; @@ -396,6 +424,12 @@ curl_version_info_data *curl_version_info(CURLversion stamp) version_info.libssh_version = ssh_buffer; #endif +#ifdef HAVE_BROTLI + version_info.brotli_ver_num = BrotliDecoderVersion(); + brotli_version(brotli_buffer, sizeof brotli_buffer); + version_info.brotli_version = brotli_buffer; +#endif + (void)stamp; /* avoid compiler warnings, we don't use this */ initialized = true; -- cgit v1.2.3