aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDaniel Stenberg <daniel@haxx.se>2005-08-24 10:57:28 +0000
committerDaniel Stenberg <daniel@haxx.se>2005-08-24 10:57:28 +0000
commita4773fcbbbf42a25c1037573fbab58aa275b9ed1 (patch)
tree965b924386052986179eb3717aeab0d82744bd54
parent1e038c4bc6ecc43bdbbe0e66a70001c7fe967bf6 (diff)
Toby Peterson added CURLOPT_IGNORE_CONTENT_LENGTH to the library, accessible
from the command line tool with --ignore-content-length. This will make it easier to download files from Apache 1.x (and similar) servers that are still having problems serving files larger than 2 or 4 GB. When this option is enabled, curl will simply have to wait for the server to close the connection to signal end of transfer. I wrote test case 269 that runs a simple test that this works.
-rw-r--r--CHANGES8
-rw-r--r--RELEASE-NOTES7
-rw-r--r--docs/curl.15
-rw-r--r--docs/libcurl/curl_easy_setopt.35
-rw-r--r--include/curl/curl.h3
-rw-r--r--lib/transfer.c2
-rw-r--r--lib/url.c4
-rw-r--r--lib/urldata.h1
-rw-r--r--src/main.c9
-rw-r--r--tests/data/Makefile.am2
-rw-r--r--tests/data/test26951
11 files changed, 92 insertions, 5 deletions
diff --git a/CHANGES b/CHANGES
index 003d4f872..302fafa65 100644
--- a/CHANGES
+++ b/CHANGES
@@ -8,6 +8,14 @@
Daniel (24 August 2005)
+- Toby Peterson added CURLOPT_IGNORE_CONTENT_LENGTH to the library, accessible
+ from the command line tool with --ignore-content-length. This will make it
+ easier to download files from Apache 1.x (and similar) servers that are
+ still having problems serving files larger than 2 or 4 GB. When this option
+ is enabled, curl will simply have to wait for the server to close the
+ connection to signal end of transfer. I wrote test case 269 that runs a
+ simple test that this works.
+
- (Trying hard to exclude emotions now.) valgrind version 3 suddenly renamed
the --logfile command line option to --log-file, and thus the test script
valgrind autodetection now has yet another version check to do and then it
diff --git a/RELEASE-NOTES b/RELEASE-NOTES
index d15c97436..fb224f787 100644
--- a/RELEASE-NOTES
+++ b/RELEASE-NOTES
@@ -2,8 +2,8 @@ Curl and libcurl 7.14.1
Public curl release number: 89
Releases counted from the very beginning: 116
- Available command line options: 107
- Available curl_easy_setopt() options: 122
+ Available command line options: 108
+ Available curl_easy_setopt() options: 123
Number of public functions in libcurl: 46
Amount of public web site mirrors: 25
Number of known libcurl bindings: 31
@@ -11,6 +11,7 @@ Curl and libcurl 7.14.1
This release includes the following changes:
+ o --ignore-content-length and CURLOPT_IGNORE_CONTENT_LENGTH added
o negotiates data connection SSL earlier when doing FTPS with PASV
o CURLOPT_COOKIELIST and CURLINFO_COOKIELIST
o trailer support for chunked encoded data streams
@@ -64,6 +65,6 @@ advice from friends like these:
Tupone Alfredo, Gisle Vanem, David Shaw, Andrew Bushnell, Dan Fandrich,
Adrian Schuur, Diego Casorran, Peteris Krumins, Jon Grubbs, Christopher
R. Palmer, Mario Schroeder, Richard Clayton, James Bursa, Jeff Pohlmeyer,
- Norbert Novotny
+ Norbert Novotny, Toby Peterson
Thanks! (and sorry if I forgot to mention someone)
diff --git a/docs/curl.1 b/docs/curl.1
index 4625cd664..a8b6380b3 100644
--- a/docs/curl.1
+++ b/docs/curl.1
@@ -429,6 +429,11 @@ for you.
See also the \fI-A/--user-agent\fP and \fI-e/--referer\fP options.
This option can be used multiple times to add/replace/remove multiple headers.
+.IP "--ignore-content-length"
+(HTTP)
+Ignore the Content-Length header. This is particularly useful for servers
+running Apache 1.x, which will report incorrect Content-Length for files
+larger than 2 gigabytes.
.IP "-i/--include"
(HTTP)
Include the HTTP-header in the output. The HTTP-header includes things
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 0b2649e1e..07f6f09e5 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -687,6 +687,11 @@ it thinks fit.
Enforce HTTP 1.0 requests.
.IP CURL_HTTP_VERSION_1_1
Enforce HTTP 1.1 requests.
+.IP CURLOPT_IGNORE_CONTENT_LENGTH
+Ignore the Content-Length header. This is useful for Apache 1.x which will
+report incorrect content length for files over 2 gigabytes. If this option
+is used, curl will not be able to accurately report progress, and will
+simply stop the download when the server ends the connection.
.RE
.SH FTP OPTIONS
.IP CURLOPT_FTPPORT
diff --git a/include/curl/curl.h b/include/curl/curl.h
index cd9f37f69..e882bd4c6 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -893,6 +893,9 @@ typedef enum {
/* feed cookies into cookie engine */
CINIT(COOKIELIST, OBJECTPOINT, 135),
+ /* ignore Content-Length */
+ CINIT(IGNORE_CONTENT_LENGTH, LONG, 136),
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
diff --git a/lib/transfer.c b/lib/transfer.c
index 8f885f5d9..d08fa594d 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -718,7 +718,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
the header completely if we get a 416 response as then we're
resuming a document that we don't get, and this header contains
info about the true size of the document we didn't get now. */
- if (!k->ignorecl &&
+ if (!k->ignorecl && !data->set.ignorecl &&
checkprefix("Content-Length:", k->p)) {
contentlength = curlx_strtoofft(k->p+15, NULL, 10);
if (data->set.max_filesize &&
diff --git a/lib/url.c b/lib/url.c
index f245a1986..92d81caa4 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -1443,6 +1443,10 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
data->set.ftp_account = va_arg(param, char *);
break;
+ case CURLOPT_IGNORE_CONTENT_LENGTH:
+ data->set.ignorecl = va_arg(param, long)?TRUE:FALSE;
+ break;
+
default:
/* unknown tag and its companion, just ignore: */
result = CURLE_FAILED_INIT; /* correct this */
diff --git a/lib/urldata.h b/lib/urldata.h
index 4200c3818..3afc90a4c 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1068,6 +1068,7 @@ struct UserDefined {
bool global_dns_cache; /* subject for future removal */
bool tcp_nodelay; /* whether to enable TCP_NODELAY or not */
+ bool ignorecl; /* ignore content length */
};
/*
diff --git a/src/main.c b/src/main.c
index 3b9046d64..e0eb6bbc7 100644
--- a/src/main.c
+++ b/src/main.c
@@ -356,6 +356,8 @@ struct Configurable {
struct curl_slist *tp_postquote;
struct curl_slist *tp_prequote;
char *ftp_account; /* for ACCT */
+
+ bool ignorecl; /* --ignore-content-length */
};
#define WARN_PREFIX "Warning: "
@@ -514,6 +516,7 @@ static void help(void)
" -G/--get Send the -d data with a HTTP GET (H)",
" -h/--help This help text",
" -H/--header <line> Custom header to pass to server (H)",
+ " --ignore-content-length Ignore the HTTP Content-Length header",
" -i/--include Include protocol headers in the output (H/F)",
" -I/--head Show document info only",
" -j/--junk-session-cookies Ignore session cookies read from file (H)",
@@ -1309,6 +1312,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
{"$m", "ftp-account", TRUE},
{"$n", "proxy-anyauth", FALSE},
{"$o", "trace-time", FALSE},
+ {"$p", "ignore-content-length", FALSE},
{"0", "http1.0", FALSE},
{"1", "tlsv1", FALSE},
@@ -1702,6 +1706,9 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */
case 'o': /* --trace-time */
config->tracetime ^= TRUE;
break;
+ case 'p': /* --ignore-content-length */
+ config->ignorecl ^= TRUE;
+ break;
}
break;
case '#': /* --progress-bar */
@@ -3896,6 +3903,8 @@ operate(struct Configurable *config, int argc, char *argv[])
curl_easy_setopt(curl, CURLOPT_SOURCE_QUOTE, config->tp_quote);
curl_easy_setopt(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account);
+ curl_easy_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl);
+
retry_numretries = config->req_retry;
retrystart = curlx_tvnow();
diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am
index 09027962e..416c2bb60 100644
--- a/tests/data/Makefile.am
+++ b/tests/data/Makefile.am
@@ -33,4 +33,4 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46 \
test237 test238 test239 test243 test245 test246 test247 test248 test249 \
test250 test251 test252 test253 test254 test255 test521 test522 test523 \
test256 test257 test258 test259 test260 test261 test262 test263 test264 \
- test265 test266 test267 test268
+ test265 test266 test267 test268 test269
diff --git a/tests/data/test269 b/tests/data/test269
new file mode 100644
index 000000000..1fbf90a3c
--- /dev/null
+++ b/tests/data/test269
@@ -0,0 +1,51 @@
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK swsclose
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Accept-Ranges: bytes
+Content-Length: 677654
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+muahahaha
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+HTTP --ignore-content-length
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/269 --ignore-content-length
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+GET /269 HTTP/1.1
+Host: 127.0.0.1:%HTTPPORT
+Accept: */*
+
+</protocol>
+</verify>