http: Implement trailing headers for chunked transfers

This adds the CURLOPT_TRAILERDATA and CURLOPT_TRAILERFUNCTION
options that allow a callback based approach to sending trailing headers
with chunked transfers.

The test server (sws) was updated to take into account the detection of the
end of transfer in the case of trailing headers presence.

Test 1591 checks that trailing headers can be sent using libcurl.

Closes #3350

6 files changed, 197 insertions, 1 deletions

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 250aa2004..52916c983 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -179,6 +179,7 @@ test1550 test1551 test1552 test1553 test1554 test1555 test1556 test1557 \
 test1560 test1561 \
 \
 test1590 \
+test1591 \
 test1600 test1601 test1602 test1603 test1604 test1605 test1606 test1607 \
 test1608 test1609 test1620 \
 \
diff --git a/tests/data/test1591 b/tests/data/test1591
new file mode 100644
index 000000000..e864fdbaa
--- /dev/null
+++ b/tests/data/test1591
@@ -0,0 +1,63 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP PUT
+CURLOPT_HTTPTRAILER_FUNCTION
+CURLOPT_HTTPTRAILER_DATA
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.0 200 OK swsclose
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+</data>
+</reply>
+# Client-side
+<client>
+<features>
+HTTP
+</features>
+<server>
+http
+</server>
+<name>
+HTTP PUT with trailers at the end
+</name>
+<tool>
+lib1591
+</tool>
+<command>
+http://%HOSTIP:%HTTPPORT/bzz/1591
+</command>
+<stdin>
+more than one byte
+</stdin>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+PUT /bzz/1591 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Transfer-Encoding: chunked

+Trailer: my-super-awesome-trailer, my-other-awesome-trailer

+Expect: 100-continue

+

+e

+Hello Cloud!

+

+0

+my-super-awesome-trailer: trail1

+my-other-awesome-trailer: trail2

+

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc
index 080421b26..4c41fe7a1 100644
--- a/tests/libtest/Makefile.inc
+++ b/tests/libtest/Makefile.inc
@@ -31,6 +31,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib1540 \
  lib1550 lib1551 lib1552 lib1553 lib1554 lib1555 lib1556 lib1557 \
  lib1560 \
+ lib1591 \
  lib1900 \
  lib2033
 
@@ -518,6 +519,10 @@ lib1557_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1557
 lib1560_SOURCES = lib1560.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1560_LDADD = $(TESTUTIL_LIBS)
 
+lib1591_SOURCES = lib1591.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib1591_LDADD = $(TESTUTIL_LIBS)
+lib1591_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1591
+
 lib1900_SOURCES = lib1900.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1900_LDADD = $(TESTUTIL_LIBS)
 lib1900_CPPFLAGS = $(AM_CPPFLAGS)
diff --git a/tests/libtest/lib1591.c b/tests/libtest/lib1591.c
new file mode 100644
index 000000000..7aba22337
--- /dev/null
+++ b/tests/libtest/lib1591.c
@@ -0,0 +1,106 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+
+/*
+ * This unit test PUT http data over proxy. Proxy header will be different
+ * from server http header
+ */
+
+#include "test.h"
+#include <stdio.h>
+#include "memdebug.h"
+
+static char data [] = "Hello Cloud!\r\n";
+static size_t consumed = 0;
+
+static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+  size_t  amount = nmemb * size; /* Total bytes curl wants */
+
+  if(consumed == strlen(data)) {
+    return 0;
+  }
+
+  if(amount > strlen(data)-consumed) {
+    amount = strlen(data);
+  }
+
+  consumed += amount;
+  (void)stream;
+  memcpy(ptr, data, amount);
+  return amount;
+}
+
+static int trailers_callback(struct curl_slist **list, void *userdata)
+{
+  (void)userdata;
+  *list = curl_slist_append(*list, "my-super-awesome-trailer: trail1");
+  *list = curl_slist_append(*list, "my-other-awesome-trailer: trail2");
+  return CURL_TRAILERFUNC_OK;
+}
+
+int test(char *URL)
+{
+  CURL *curl = NULL;
+  CURLcode res = CURLE_FAILED_INIT;
+  /* http and proxy header list*/
+  struct curl_slist *hhl = NULL;
+
+  if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) {
+    fprintf(stderr, "curl_global_init() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+
+  curl = curl_easy_init();
+  if(!curl) {
+    fprintf(stderr, "curl_easy_init() failed\n");
+    curl_global_cleanup();
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  hhl = curl_slist_append(hhl, "Trailer: my-super-awesome-trailer,"
+                               " my-other-awesome-trailer");
+  if(!hhl) {
+    goto test_cleanup;
+  }
+
+  test_setopt(curl, CURLOPT_URL, URL);
+  test_setopt(curl, CURLOPT_HTTPHEADER, hhl);
+  test_setopt(curl, CURLOPT_PUT, 1L);
+  test_setopt(curl, CURLOPT_READFUNCTION, read_callback);
+  test_setopt(curl, CURLOPT_TRAILERFUNCTION, trailers_callback);
+  test_setopt(curl, CURLOPT_TRAILERDATA, NULL);
+
+  res = curl_easy_perform(curl);
+
+test_cleanup:
+
+  curl_easy_cleanup(curl);
+
+  curl_slist_free_all(hhl);
+
+  curl_global_cleanup();
+
+  return (int)res;
+}
+
diff --git a/tests/libtest/mk-lib1521.pl b/tests/libtest/mk-lib1521.pl
index 9771cf57f..fb1401a53 100755
--- a/tests/libtest/mk-lib1521.pl
+++ b/tests/libtest/mk-lib1521.pl
@@ -121,6 +121,7 @@ static int geterr(const char *name, CURLcode val, int lineno)
 static curl_progress_callback progresscb;
 static curl_write_callback headercb;
 static curl_debug_callback debugcb;
+static curl_trailer_callback trailercb;
 static curl_ssl_ctx_callback ssl_ctx_cb;
 static curl_ioctl_callback ioctlcb;
 static curl_sockopt_callback sockoptcb;
diff --git a/tests/server/sws.c b/tests/server/sws.c
index cf3d291d9..87c0204c9 100644
--- a/tests/server/sws.c
+++ b/tests/server/sws.c
@@ -253,6 +253,9 @@ SIG_ATOMIC_T got_exit_signal = 0;
 
 static volatile int exit_signal = 0;
 
+/* work around for handling trailing headers */
+static int already_recv_zeroed_chunk = FALSE;
+
 /* signal handler that will be triggered to indicate that the program
   should finish its execution in a controlled manner as soon as possible.
   The first time this is called it will set got_exit_signal to one and
@@ -755,10 +758,27 @@ static int ProcessRequest(struct httprequest *req)
       chunked = TRUE;
     }
 
+
     if(chunked) {
-      if(strstr(req->reqbuf, "\r\n0\r\n\r\n"))
+      if(strstr(req->reqbuf, "\r\n0\r\n\r\n")) {
         /* end of chunks reached */
         return 1; /* done */
+      }
+      else if(strstr(req->reqbuf, "\r\n0\r\n")) {
+        char *last_crlf_char = strstr(req->reqbuf, "\r\n\r\n");
+        while(TRUE) {
+          if(!strstr(last_crlf_char + 4, "\r\n\r\n"))
+            break;
+          last_crlf_char = strstr(last_crlf_char + 4, "\r\n\r\n");
+        }
+        if(last_crlf_char &&
+           last_crlf_char > strstr(req->reqbuf, "\r\n0\r\n"))
+          return 1;
+        already_recv_zeroed_chunk = TRUE;
+        return 0;
+      }
+      else if(already_recv_zeroed_chunk && strstr(req->reqbuf, "\r\n\r\n"))
+        return 1;
       else
         return 0; /* not done */
     }