New libcurl option to keep sending on error

Add the new option CURLOPT_KEEP_SENDING_ON_ERROR to control whether
sending the request body shall be completed when the server responds
early with an error status code.

This is suitable for manual NTLM authentication.

Reviewed-by: Jay Satiro

Closes https://github.com/curl/curl/pull/904

14 files changed, 364 insertions, 7 deletions

diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 75f1ce46b..b32e1c03b 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -144,6 +144,8 @@ Error message buffer. See \fICURLOPT_ERRORBUFFER(3)\fP
 stderr replacement stream. See \fICURLOPT_STDERR(3)\fP
 .IP CURLOPT_FAILONERROR
 Fail on HTTP 4xx errors. \fICURLOPT_FAILONERROR(3)\fP
+.IP CURLOPT_KEEP_SENDING_ON_ERROR
+Keep sending on HTTP >= 300 errors. \fICURLOPT_KEEP_SENDING_ON_ERROR(3)\fP
 .SH NETWORK OPTIONS
 .IP CURLOPT_URL
 URL to work on. See \fICURLOPT_URL(3)\fP
diff --git a/docs/libcurl/opts/CURLOPT_FAILONERROR.3 b/docs/libcurl/opts/CURLOPT_FAILONERROR.3
index 79474cefd..93d8ba6ed 100644
--- a/docs/libcurl/opts/CURLOPT_FAILONERROR.3
+++ b/docs/libcurl/opts/CURLOPT_FAILONERROR.3
@@ -53,4 +53,4 @@ Along with HTTP
 .SH RETURN VALUE
 Returns CURLE_OK if HTTP is enabled, and CURLE_UNKNOWN_OPTION if not.
 .SH "SEE ALSO"
-.BR CURLOPT_HTTP200ALIASES "(3), "
+.BR CURLOPT_HTTP200ALIASES "(3), " CURLOPT_KEEP_SENDING_ON_ERROR "(3), "
diff --git a/docs/libcurl/opts/CURLOPT_KEEP_SENDING_ON_ERROR.3 b/docs/libcurl/opts/CURLOPT_KEEP_SENDING_ON_ERROR.3
new file mode 100644
index 000000000..4ddd8687c
--- /dev/null
+++ b/docs/libcurl/opts/CURLOPT_KEEP_SENDING_ON_ERROR.3
@@ -0,0 +1,52 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_KEEP_SENDING_ON_ERROR 3 "22 Sep 2016" "libcurl 7.51.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_KEEP_SENDING_ON_ERROR \- keep sending on early HTTP response >= 300
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_KEEP_SENDING_ON_ERROR, long keep_sending);
+.SH DESCRIPTION
+A long parameter set to 1 tells the library to keep sending the request body
+if the HTTP code returned is equal to or larger than 300. The default action
+would be to stop sending and close the stream or connection.
+
+This option is suitable for manual NTLM authentication, i.e. if an application
+does not use \fICURLOPT_HTTPAUTH(3)\fP, but instead sets "Authorization: NTLM ..."
+headers manually using \fICURLOPT_HTTPHEADER(3)\fP.
+
+Most applications do not need this option.
+
+.SH DEFAULT
+0, stop sending on error
+.SH PROTOCOLS
+HTTP
+.SH EXAMPLE
+TODO
+.SH AVAILABILITY
+Along with HTTP
+.SH RETURN VALUE
+Returns CURLE_OK if HTTP is enabled, and CURLE_UNKNOWN_OPTION if not.
+.SH "SEE ALSO"
+.BR CURLOPT_FAILONERROR "(3), " CURLOPT_HTTPHEADER "(3), "
diff --git a/docs/libcurl/opts/Makefile.am b/docs/libcurl/opts/Makefile.am
index a3fc0647f..e67364ccc 100644
--- a/docs/libcurl/opts/Makefile.am
+++ b/docs/libcurl/opts/Makefile.am
@@ -170,6 +170,7 @@ man_MANS =                                      \
  CURLOPT_IOCTLFUNCTION.3                        \
  CURLOPT_IPRESOLVE.3                            \
  CURLOPT_ISSUERCERT.3                           \
+ CURLOPT_KEEP_SENDING_ON_ERROR.3                \
  CURLOPT_KEYPASSWD.3                            \
  CURLOPT_KRBLEVEL.3                             \
  CURLOPT_LOCALPORT.3                            \
@@ -457,6 +458,7 @@ HTMLPAGES =                                     \
  CURLOPT_IOCTLFUNCTION.html                     \
  CURLOPT_IPRESOLVE.html                         \
  CURLOPT_ISSUERCERT.html                        \
+ CURLOPT_KEEP_SENDING_ON_ERROR.html             \
  CURLOPT_KEYPASSWD.html                         \
  CURLOPT_KRBLEVEL.html                          \
  CURLOPT_LOCALPORT.html                         \
@@ -744,6 +746,7 @@ PDFPAGES =                                      \
  CURLOPT_IOCTLFUNCTION.pdf                      \
  CURLOPT_IPRESOLVE.pdf                          \
  CURLOPT_ISSUERCERT.pdf                         \
+ CURLOPT_KEEP_SENDING_ON_ERROR.pdf              \
  CURLOPT_KEYPASSWD.pdf                          \
  CURLOPT_KRBLEVEL.pdf                           \
  CURLOPT_LOCALPORT.pdf                          \
diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index e61319546..f6365ae11 100644
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -411,6 +411,7 @@ CURLOPT_IOCTLFUNCTION           7.12.3
 CURLOPT_IPRESOLVE               7.10.8
 CURLOPT_ISSUERCERT              7.19.0
 CURLOPT_KEYPASSWD               7.17.0
+CURLOPT_KEEP_SENDING_ON_ERROR   7.51.0
 CURLOPT_KRB4LEVEL               7.3           7.17.0
 CURLOPT_KRBLEVEL                7.16.4
 CURLOPT_LOCALPORT               7.15.2
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 0ac238c28..9c09cb966 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -1700,6 +1700,10 @@ typedef enum {
   /* Set TCP Fast Open */
   CINIT(TCP_FASTOPEN, LONG, 244),
 
+  /* Continue to send data if the server responds early with an
+   * HTTP status code >= 300 */
+  CINIT(KEEP_SENDING_ON_ERROR, LONG, 245),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 
diff --git a/lib/http.c b/lib/http.c
index 07ad463c9..65c145a13 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -3181,12 +3181,21 @@ CURLcode Curl_http_readwrite_headers(struct Curl_easy *data,
              * connection for closure after we've read the entire response.
              */
             if(!k->upload_done) {
-              infof(data, "HTTP error before end of send, stop sending\n");
-              streamclose(conn, "Stop sending data before everything sent");
-              k->upload_done = TRUE;
-              k->keepon &= ~KEEP_SEND; /* don't send */
-              if(data->state.expect100header)
-                k->exp100 = EXP100_FAILED;
+              if(data->set.http_keep_sending_on_error) {
+                infof(data, "HTTP error before end of send, keep sending\n");
+                if(k->exp100 > EXP100_SEND_DATA) {
+                  k->exp100 = EXP100_SEND_DATA;
+                  k->keepon |= KEEP_SEND;
+                }
+              }
+              else {
+                infof(data, "HTTP error before end of send, stop sending\n");
+                streamclose(conn, "Stop sending data before everything sent");
+                k->upload_done = TRUE;
+                k->keepon &= ~KEEP_SEND; /* don't send */
+                if(data->state.expect100header)
+                  k->exp100 = EXP100_FAILED;
+              }
             }
             break;
 
diff --git a/lib/url.c b/lib/url.c
index f355c7a22..74e9bf5c6 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -782,6 +782,10 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
      */
     data->set.http_fail_on_error = (0 != va_arg(param, long)) ? TRUE : FALSE;
     break;
+  case CURLOPT_KEEP_SENDING_ON_ERROR:
+    data->set.http_keep_sending_on_error = (0 != va_arg(param, long)) ?
+                                           TRUE : FALSE;
+    break;
   case CURLOPT_UPLOAD:
   case CURLOPT_PUT:
     /*
diff --git a/lib/urldata.h b/lib/urldata.h
index d5efe2a97..fd9870e26 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1611,6 +1611,7 @@ struct UserDefined {
   bool ftp_use_port;     /* use the FTP PORT command */
   bool hide_progress;    /* don't use the progress meter */
   bool http_fail_on_error;  /* fail on HTTP error codes >= 400 */
+  bool http_keep_sending_on_error; /* for HTTP status codes >= 300 */
   bool http_follow_location; /* follow HTTP redirects */
   bool http_transfer_encoding; /* request compressed HTTP transfer-encoding */
   bool http_disable_hostname_check_before_authentication;
diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
index 486f655e9..4176905a0 100644
--- a/packages/OS400/curl.inc.in
+++ b/packages/OS400/curl.inc.in
@@ -1254,6 +1254,8 @@
      d                 c                   10243
      d  CURLOPT_TCP_FASTOPEN...
      d                 c                   00244
+     d  CURLOPT_KEEP_SENDING_ON_ERROR...
+     d                 c                   00245
       *
       /if not defined(CURL_NO_OLDIES)
      d  CURLOPT_FILE   c                   10001
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index a3d315abb..8b402f02c 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -157,6 +157,7 @@ test1516 test1517 \
 test1520 \
 \
 test1525 test1526 test1527 test1528 test1529 test1530 test1531 test1532 \
+test1533 \
 \
 test1600 test1601 test1602 test1603 test1604 test1605 \
 \
diff --git a/tests/data/test1533 b/tests/data/test1533
new file mode 100644
index 000000000..56518161c
--- /dev/null
+++ b/tests/data/test1533
@@ -0,0 +1,74 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP POST
+CURLOPT_KEEP_SENDING_ON_ERROR
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<servercmd>
+auth_required
+</servercmd>
+<data nocheck="yes">
+HTTP/1.1 401 Authorization Required

+Date: Thu, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake

+Content-Length: 15

+
+Early Response
+</data>
+</reply>
+# Client-side
+<client>
+<server>
+http
+</server>
+<tool>
+lib1533
+</tool>
+<name>
+HTTP with CURLOPT_KEEP_SENDING_ON_ERROR and an early error response
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/1533
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+# TEST_ERR_SUCCESS is errorcode 120
+<verify>
+<errorcode>
+120
+</errorcode>
+<protocol nonewline="yes">
+POST /1533 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Content-Length: 3

+Content-Type: application/x-www-form-urlencoded

+

+POST /1533 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Content-Length: 3

+Content-Type: application/x-www-form-urlencoded

+

+POST /1533 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Content-Length: 3

+Content-Type: application/x-www-form-urlencoded

+

+aaaPOST /1533 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Content-Length: 3

+Content-Type: application/x-www-form-urlencoded

+

+aaa
+</protocol>
+</verify>
+</testcase>
diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc
index 6a9088c33..da905706b 100644
--- a/tests/libtest/Makefile.inc
+++ b/tests/libtest/Makefile.inc
@@ -24,6 +24,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib1509 lib1510 lib1511 lib1512 lib1513 lib1514 lib1515         lib1517 \
  lib1520 \
  lib1525 lib1526 lib1527 lib1528 lib1529 lib1530 lib1531 lib1532 \
+ lib1533 \
  lib1900 \
  lib2033
 
@@ -395,6 +396,10 @@ lib1532_SOURCES = lib1532.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1532_LDADD = $(TESTUTIL_LIBS)
 lib1532_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1532
 
+lib1533_SOURCES = lib1533.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib1533_LDADD = $(TESTUTIL_LIBS)
+lib1533_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1533
+
 lib1900_SOURCES = lib1900.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1900_LDADD = $(TESTUTIL_LIBS)
 lib1900_CPPFLAGS = $(AM_CPPFLAGS)
diff --git a/tests/libtest/lib1533.c b/tests/libtest/lib1533.c
new file mode 100644
index 000000000..de403e195
--- /dev/null
+++ b/tests/libtest/lib1533.c
@@ -0,0 +1,199 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+
+/*
+ * This test sends data with CURLOPT_KEEP_SENDING_ON_ERROR.
+ * The server responds with an early error response.
+ * The test is successful if the connection can be reused for the next request,
+ * because this implies that the data has been sent completely to the server.
+ */
+
+#include "test.h"
+
+#include "memdebug.h"
+
+struct cb_data {
+  CURL *easy_handle;
+  int response_received;
+  int paused;
+  size_t remaining_bytes;
+};
+
+
+static void reset_data(struct cb_data *data, CURL *curl)
+{
+  data->easy_handle = curl;
+  data->response_received = 0;
+  data->paused = 0;
+  data->remaining_bytes = 3;
+}
+
+
+static size_t read_callback(void *ptr, size_t size, size_t nitems,
+                            void *userdata)
+{
+  struct cb_data *data = (struct cb_data *)userdata;
+
+  /* wait until the server has sent all response headers */
+  if(data->response_received) {
+    size_t totalsize = nitems * size;
+
+    size_t bytes_to_send = data->remaining_bytes;
+    if(bytes_to_send > totalsize) {
+      bytes_to_send = totalsize;
+    }
+
+    memset(ptr, 'a', bytes_to_send);
+    data->remaining_bytes -= bytes_to_send;
+
+    return bytes_to_send;
+  }
+  else {
+    data->paused = 1;
+    return CURL_READFUNC_PAUSE;
+  }
+}
+
+
+static size_t write_callback(char *ptr, size_t size, size_t nmemb,
+                             void *userdata)
+{
+  struct cb_data *data = (struct cb_data *)userdata;
+  size_t totalsize = nmemb * size;
+
+  /* unused parameter */
+  (void)ptr;
+
+  /* all response headers have been received */
+  data->response_received = 1;
+
+  if(data->paused) {
+    /* continue to send request body data */
+    data->paused = 0;
+    curl_easy_pause(data->easy_handle, CURLPAUSE_CONT);
+  }
+
+  return totalsize;
+}
+
+
+static int perform_and_check_connections(CURL *curl, const char *description,
+                                         long expected_connections)
+{
+  CURLcode res;
+  long connections = 0;
+
+  res = curl_easy_perform(curl);
+  if(res != CURLE_OK) {
+    fprintf(stderr, "curl_easy_perform() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  res = curl_easy_getinfo(curl, CURLINFO_NUM_CONNECTS, &connections);
+  if(res != CURLE_OK) {
+    fprintf(stderr, "curl_easy_getinfo() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  fprintf(stderr, "%s: expected: %ld connections; actual: %ld connections\n",
+          description, expected_connections, connections);
+
+  if(connections != expected_connections) {
+    return TEST_ERR_FAILURE;
+  }
+
+  return TEST_ERR_SUCCESS;
+}
+
+
+int test(char *URL)
+{
+  struct cb_data data;
+  CURL *curl = NULL;
+  CURLcode res = CURLE_FAILED_INIT;
+
+  if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) {
+    fprintf(stderr, "curl_global_init() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  if((curl = curl_easy_init()) == NULL) {
+    fprintf(stderr, "curl_easy_init() failed\n");
+    curl_global_cleanup();
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  reset_data(&data, curl);
+
+  test_setopt(curl, CURLOPT_URL, URL);
+  test_setopt(curl, CURLOPT_POST, 1L);
+  test_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE,
+              (curl_off_t)data.remaining_bytes);
+  test_setopt(curl, CURLOPT_VERBOSE, 1L);
+  test_setopt(curl, CURLOPT_READFUNCTION, read_callback);
+  test_setopt(curl, CURLOPT_READDATA, &data);
+  test_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+  test_setopt(curl, CURLOPT_WRITEDATA, &data);
+
+  res = perform_and_check_connections(curl,
+    "First request without CURLOPT_KEEP_SENDING_ON_ERROR", 1);
+  if(res != TEST_ERR_SUCCESS) {
+    goto test_cleanup;
+  }
+
+  reset_data(&data, curl);
+
+  res = perform_and_check_connections(curl,
+    "Second request without CURLOPT_KEEP_SENDING_ON_ERROR", 1);
+  if(res != TEST_ERR_SUCCESS) {
+    goto test_cleanup;
+  }
+
+  test_setopt(curl, CURLOPT_KEEP_SENDING_ON_ERROR, 1L);
+
+  reset_data(&data, curl);
+
+  res = perform_and_check_connections(curl,
+    "First request with CURLOPT_KEEP_SENDING_ON_ERROR", 1);
+  if(res != TEST_ERR_SUCCESS) {
+    goto test_cleanup;
+  }
+
+  reset_data(&data, curl);
+
+  res = perform_and_check_connections(curl,
+    "Second request with CURLOPT_KEEP_SENDING_ON_ERROR", 0);
+  if(res != TEST_ERR_SUCCESS) {
+    goto test_cleanup;
+  }
+
+  res = TEST_ERR_SUCCESS;
+
+test_cleanup:
+
+  curl_easy_cleanup(curl);
+
+  curl_global_cleanup();
+
+  return (int)res;
+}
+