From 18e5cb77e986911063da8ab6bf254d632b2de6ea Mon Sep 17 00:00:00 2001 From: Maros Priputen Date: Wed, 30 Oct 2019 09:43:14 +0100 Subject: curl: two new command line options for etags --etag-compare and --etag-save Suggested-by: Paul Hoffman Fixes #4277 Closes #4543 --- docs/cmdline-opts/Makefile.inc | 2 + docs/cmdline-opts/etag-compare.d | 17 +++++++ docs/cmdline-opts/etag-save.d | 15 ++++++ src/tool_cb_hdr.c | 54 +++++++++++++++++++++ src/tool_cb_hdr.h | 3 +- src/tool_cfgable.c | 2 + src/tool_cfgable.h | 2 + src/tool_getparam.c | 10 ++++ src/tool_help.c | 4 ++ src/tool_operate.c | 101 +++++++++++++++++++++++++++++++++++++++ src/tool_operate.h | 1 + tests/data/Makefile.inc | 3 +- tests/data/test339 | 63 ++++++++++++++++++++++++ tests/data/test341 | 57 ++++++++++++++++++++++ tests/data/test342 | 59 +++++++++++++++++++++++ 15 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 docs/cmdline-opts/etag-compare.d create mode 100644 docs/cmdline-opts/etag-save.d create mode 100644 tests/data/test339 create mode 100644 tests/data/test341 create mode 100644 tests/data/test342 diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index fd29dfb23..829551ff6 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -38,6 +38,8 @@ DPAGES = \ dump-header.d \ egd-file.d \ engine.d \ + etag-save.d \ + etag-compare.d \ expect100-timeout.d \ fail-early.d \ fail.d \ diff --git a/docs/cmdline-opts/etag-compare.d b/docs/cmdline-opts/etag-compare.d new file mode 100644 index 000000000..1c1364f43 --- /dev/null +++ b/docs/cmdline-opts/etag-compare.d @@ -0,0 +1,17 @@ +Long: etag-compare +Arg: +Help: Pass an ETag from a file as a custom header +Protocols: HTTP +--- +This option makes a conditional HTTP request for the specific +ETag read from the given file by sending a custom If-None-Match +header using the extracted ETag. + +For correct results, make sure that specified file contains only a single +line with a desired ETag. An empty file is parsed as an empty ETag. + +Use the option --etag-save to first save the ETag from a response, and +then use this option to compare using the saved ETag in a subsequent request. + +\fCOMPARISON\fP: There are 2 types of comparison or ETags, Weak and Strong. +This option expects, and uses a strong comparison. diff --git a/docs/cmdline-opts/etag-save.d b/docs/cmdline-opts/etag-save.d new file mode 100644 index 000000000..fa0694d14 --- /dev/null +++ b/docs/cmdline-opts/etag-save.d @@ -0,0 +1,15 @@ +Long: etag-save +Arg: +Help: Parse ETag from a request and save it to a file +Protocols: HTTP +--- +This option saves an HTTP ETag to the specified file. Etag is +usually part of headers returned by a request. When server sends an +ETag, it must be enveloped by a double quote. This option extracts the +ETag without the double quotes and saves it into the . + +A server can send a week ETag which is prefixed by "W/". This identifier +is not considered, and only relevant ETag between quotation marks is parsed. + +It an ETag wasn't send by the server or it cannot be parsed, and empty +file is created. diff --git a/src/tool_cb_hdr.c b/src/tool_cb_hdr.c index b0880f186..77224adba 100644 --- a/src/tool_cb_hdr.c +++ b/src/tool_cb_hdr.c @@ -59,6 +59,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata) struct HdrCbData *hdrcbdata = &per->hdrcbdata; struct OutStruct *outs = &per->outs; struct OutStruct *heads = &per->heads; + struct OutStruct *etag_save = &per->etag_save; const char *str = ptr; const size_t cb = size * nmemb; const char *end = (char *)ptr + cb; @@ -95,6 +96,59 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata) (void)fflush(heads->stream); } + /* + * Write etag to file when --etag-save option is given. + * etag string that we want is enveloped in double quotes + */ + if(etag_save->config->etag_save_file && etag_save->stream) { + /* match only header that start with etag (case insensitive) */ + if(curl_strnequal(str, "etag:", 5)) { + char *etag_h = NULL; + char *first = NULL; + char *last = NULL; + size_t etag_length = 0; + + etag_h = ptr; + /* point to first occurence of double quote */ + first = memchr(etag_h, '\"', cb); + + /* + * if server side messed with the etag header and doesn't include + * double quotes around the etag, kindly exit with a warning + */ + + if(!first) { + warnf( + etag_save->config->global, + "\nReceived header etag is missing double quote/s\n"); + return 1; + } + else { + /* discard first double quote */ + first++; + } + + /* point to last occurence of double quote */ + last = memchr(first, '\"', cb); + + if(!last) { + warnf( + etag_save->config->global, + "\nReceived header etag is missing double quote/s\n"); + return 1; + } + + /* get length of desired etag */ + etag_length = (size_t)last - (size_t)first; + + fwrite(first, size, etag_length, etag_save->stream); + /* terminate with new line */ + fputc('\n', etag_save->stream); + } + + (void)fflush(etag_save->stream); + } + /* * This callback sets the filename where output shall be written when * curl options --remote-name (-O) and --remote-header-name (-J) have diff --git a/src/tool_cb_hdr.h b/src/tool_cb_hdr.h index cf544dfcb..ec5772f55 100644 --- a/src/tool_cb_hdr.h +++ b/src/tool_cb_hdr.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2018, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2019, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -43,6 +43,7 @@ struct HdrCbData { struct OperationConfig *config; struct OutStruct *outs; struct OutStruct *heads; + struct OutStruct *etag_save; bool honor_cd_filename; }; diff --git a/src/tool_cfgable.c b/src/tool_cfgable.c index efa8c50b2..f802a5a31 100644 --- a/src/tool_cfgable.c +++ b/src/tool_cfgable.c @@ -128,6 +128,8 @@ static void free_config_fields(struct OperationConfig *config) Curl_safefree(config->pubkey); Curl_safefree(config->hostpubmd5); Curl_safefree(config->engine); + Curl_safefree(config->etag_save_file); + Curl_safefree(config->etag_compare_file); Curl_safefree(config->request_target); Curl_safefree(config->customrequest); Curl_safefree(config->krblevel); diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 4372cc6fc..32e811eaa 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -156,6 +156,8 @@ struct OperationConfig { char *pubkey; char *hostpubmd5; char *engine; + char *etag_save_file; + char *etag_compare_file; bool crlf; char *customrequest; char *krblevel; diff --git a/src/tool_getparam.c b/src/tool_getparam.c index 75faff34d..3efc23e1e 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -268,6 +268,8 @@ static const struct LongShort aliases[]= { {"E9", "proxy-tlsv1", ARG_NONE}, {"EA", "socks5-basic", ARG_BOOL}, {"EB", "socks5-gssapi", ARG_BOOL}, + {"EC", "etag-save", ARG_FILENAME}, + {"ED", "etag-compare", ARG_FILENAME}, {"f", "fail", ARG_BOOL}, {"fa", "fail-early", ARG_BOOL}, {"fb", "styled-output", ARG_BOOL}, @@ -1697,6 +1699,14 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ config->socks5_auth &= ~CURLAUTH_GSSAPI; break; + case 'C': + GetStr(&config->etag_save_file, nextarg); + break; + + case 'D': + GetStr(&config->etag_compare_file, nextarg); + break; + default: /* unknown flag */ return PARAM_OPTION_UNKNOWN; } diff --git a/src/tool_help.c b/src/tool_help.c index 21900108b..8d3f34547 100644 --- a/src/tool_help.c +++ b/src/tool_help.c @@ -131,6 +131,10 @@ static const struct helptxt helptext[] = { "EGD socket path for random data"}, {" --engine ", "Crypto engine to use"}, + {" --etag-save ", + "Get an ETag from response header and save it to a FILE"}, + {" --etag-compare ", + "Get an ETag from a file and send a conditional request"}, {" --expect100-timeout ", "How long to wait for 100-continue"}, {"-f, --fail", diff --git a/src/tool_operate.c b/src/tool_operate.c index 4b2ffb55b..23971f112 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -644,6 +644,12 @@ static CURLcode post_per_transfer(struct GlobalConfig *global, if(per->heads.alloc_filename) Curl_safefree(per->heads.filename); + if(per->etag_save.fopened && per->etag_save.stream) + fclose(per->etag_save.stream); + + if(per->etag_save.alloc_filename) + Curl_safefree(per->etag_save.filename); + curl_easy_cleanup(per->curl); if(outs->alloc_filename) free(outs->filename); @@ -834,6 +840,7 @@ static CURLcode single_transfer(struct GlobalConfig *global, struct OutStruct *outs; struct InStruct *input; struct OutStruct *heads; + struct OutStruct *etag_save; struct HdrCbData *hdrcbdata = NULL; CURL *curl = curl_easy_init(); result = add_per_transfer(&per); @@ -882,6 +889,99 @@ static CURLcode single_transfer(struct GlobalConfig *global, } } + /* disallowing simultaneous use of --etag-save and --etag-compare */ + if(config->etag_save_file && config->etag_compare_file) { + warnf( + config->global, + "Cannot use --etag-save and --etag-compare at the same time\n"); + + result = CURLE_UNKNOWN_OPTION; + break; + } + + /* --etag-save */ + etag_save = &per->etag_save; + etag_save->stream = stdout; + etag_save->config = config; + if(config->etag_save_file) { + /* open file for output: */ + if(strcmp(config->etag_save_file, "-")) { + FILE *newfile = fopen(config->etag_save_file, "wb"); + if(!newfile) { + warnf( + config->global, + "Failed to open %s\n", config->etag_save_file); + + result = CURLE_WRITE_ERROR; + break; + } + else { + etag_save->filename = config->etag_save_file; + etag_save->s_isreg = TRUE; + etag_save->fopened = TRUE; + etag_save->stream = newfile; + } + } + else { + /* always use binary mode for protocol header output */ + set_binmode(etag_save->stream); + } + } + + /* --etag-compare */ + if(config->etag_compare_file) { + char *etag_from_file = NULL; + char *header = NULL; + size_t file_size = 0; + + /* open file for reading: */ + FILE *file = fopen(config->etag_compare_file, FOPEN_READTEXT); + if(!file) { + warnf( + config->global, + "Failed to open %s\n", config->etag_compare_file); + + result = CURLE_READ_ERROR; + break; + } + + /* get file size */ + fseek(file, 0, SEEK_END); + file_size = ftell(file); + + /* + * check if file is empty, if it's not load etag + * else continue with empty etag + */ + if(file_size != 0) { + fseek(file, 0, SEEK_SET); + file2string(&etag_from_file, file); + + header = aprintf("If-None-Match: \"%s\"", etag_from_file); + } + else { + header = aprintf("If-None-Match: \"\""); + } + + if(!header) { + warnf( + config->global, + "Failed to allocate memory for custom etag header\n"); + + result = CURLE_OUT_OF_MEMORY; + break; + } + + /* add Etag from file to list of custom headers */ + add2list(&config->headers, header); + + Curl_safefree(header); + Curl_safefree(etag_from_file); + + if(file) { + fclose(file); + } + } hdrcbdata = &per->hdrcbdata; @@ -1769,6 +1869,7 @@ static CURLcode single_transfer(struct GlobalConfig *global, hdrcbdata->outs = outs; hdrcbdata->heads = heads; + hdrcbdata->etag_save = etag_save; hdrcbdata->global = global; hdrcbdata->config = config; diff --git a/src/tool_operate.h b/src/tool_operate.h index 7223db767..39227c0f3 100644 --- a/src/tool_operate.h +++ b/src/tool_operate.h @@ -48,6 +48,7 @@ struct per_transfer { struct ProgressData progressbar; struct OutStruct outs; struct OutStruct heads; + struct OutStruct etag_save; struct InStruct input; struct HdrCbData hdrcbdata; char errorbuffer[CURL_ERROR_SIZE]; diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index c45bced5b..f07d6739c 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -57,8 +57,7 @@ test298 test299 test300 test301 test302 test303 test304 test305 test306 \ test307 test308 test309 test310 test311 test312 test313 test314 test315 \ test316 test317 test318 test319 test320 test321 test322 test323 test324 \ test325 test326 test327 test328 test329 test330 test331 test332 test333 \ -test334 test335 test336 test337 test338 \ -test340 \ +test334 test335 test336 test337 test338 test339 test340 test341 test342 \ \ test350 test351 test352 test353 test354 test355 test356 \ test393 test394 test395 \ diff --git a/tests/data/test339 b/tests/data/test339 new file mode 100644 index 000000000..cd6e49892 --- /dev/null +++ b/tests/data/test339 @@ -0,0 +1,63 @@ + + + +HTTP +HTTP GET + + +# +# Server-side + + +HTTP/1.1 200 funky chunky! +Server: fakeit/0.9 fakeitbad/1.0 +Transfer-Encoding: chunked +Trailer: chunky-trailer +Connection: mooo +ETag: "asdf" + +40 +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +30 +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +21;heresatest=moooo +cccccccccccccccccccccccccccccccc + +0 +chunky-trailer: header data + + + + +# +# Client-side + + +http + + +Check if --etag-save saved correct etag to a file + + +http://%HOSTIP:%HTTPPORT/339 --etag-save log/etag339 + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +GET /339 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* + + + +asdf + + + + diff --git a/tests/data/test341 b/tests/data/test341 new file mode 100644 index 000000000..5e952ad98 --- /dev/null +++ b/tests/data/test341 @@ -0,0 +1,57 @@ + + + +HTTP +HTTP GET + + +# +# Server-side + + +HTTP/1.1 200 funky chunky! +Server: fakeit/0.9 fakeitbad/1.0 +Transfer-Encoding: chunked +Trailer: chunky-trailer +Connection: mooo +ETag: "asdf" + +40 +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +30 +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +21;heresatest=moooo +cccccccccccccccccccccccccccccccc + +0 +chunky-trailer: header data + + + + +# +# Client-side + + +http + + +Try to open a non existing file with --etag-compare should return an error + + +http://%HOSTIP:%HTTPPORT/341 --etag-compare log/etag341 + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +26 + + + + diff --git a/tests/data/test342 b/tests/data/test342 new file mode 100644 index 000000000..b54e04819 --- /dev/null +++ b/tests/data/test342 @@ -0,0 +1,59 @@ + + + +HTTP +HTTP GET + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Thu, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 6 +Connection: close +Content-Type: text/html +Funny-head: yesyes + +-foo- + + + +# +# Client-side + + +http + + +Check if --etag-compare set correct etag in header + + +21025-dc7-39462498 + + +http://%HOSTIP:%HTTPPORT/342 --etag-compare log/etag342 + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +GET /342 HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* +If-None-Match: "21025-dc7-39462498" + + + + -- cgit v1.2.3