diff options
| -rw-r--r-- | CHANGES | 10 | ||||
| -rw-r--r-- | RELEASE-NOTES | 15 | ||||
| -rw-r--r-- | docs/libcurl/curl_easy_setopt.3 | 23 | ||||
| -rw-r--r-- | include/curl/curl.h | 8 | ||||
| -rw-r--r-- | lib/ftp.c | 53 | ||||
| -rw-r--r-- | lib/http.c | 55 | ||||
| -rw-r--r-- | lib/transfer.c | 15 | ||||
| -rw-r--r-- | lib/url.c | 18 | ||||
| -rw-r--r-- | lib/urldata.h | 5 | ||||
| -rw-r--r-- | src/main.c | 26 |
10 files changed, 173 insertions, 55 deletions
@@ -7,6 +7,16 @@ Changelog Daniel S (10 Jan 2008) +- Georg Lippitsch brought CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA to allow + libcurl to seek in a given input stream. This is particularly important when + doing upload resumes when there's already a huge part of the file present + remotely. Before, and still if this callback isn't used, libcurl will read + and through away the entire file up to the point to where the resuming + begins (which of course can be a slow opereration depending on file size, + I/O bandwidth and more). This new function will also be preferred to get + used instead of the CURLOPT_IOCTLFUNCTION for seeking back in a stream when + doing multi-stage HTTP auth with POST/PUT. + - Nikitinskit Dmitriy filed bug report #1868255 (http://curl.haxx.se/bug/view.cgi?id=1868255) with a patch. It identifies and fixes a problem with parsing WWW-Authenticate: headers with additional diff --git a/RELEASE-NOTES b/RELEASE-NOTES index bcfb898cc..27d22dc94 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -2,7 +2,7 @@ Curl and libcurl 7.18.0 Public curl releases: 103 Command line options: 125 - curl_easy_setopt() options: 148 + curl_easy_setopt() options: 150 Public functions in libcurl: 56 Public web site mirrors: 42 Known libcurl bindings: 36 @@ -10,13 +10,14 @@ Curl and libcurl 7.18.0 This release includes the following changes: - o --data-urlencode was added - o CURLOPT_PROXY_TRANSFER_MODE was added - o --no-keep-alive was added, since starting now curl is doing connections with - keep-alive enabled by default + o --data-urlencode + o CURLOPT_PROXY_TRANSFER_MODE + o --no-keep-alive - now curl does connections with keep-alive enabled by + default o --socks4a added (proxy type CURLPROXY_SOCKS4A for libcurl) o --socks5-hostname added (CURLPROXY_SOCKS5_HOSTNAME for libcurl) - o curl_easy_pause() added + o curl_easy_pause() + o CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA This release includes the following bugfixes: @@ -69,6 +70,6 @@ advice from friends like these: Emil Romanus, Alessandro Vesely, Ray Pekowski, Spacen Jasset, Andrew Moise, Gilles Blanc, David Wright, Vikram Saxena, Mateusz Loskot, Gary Maxwell, Dmitry Kurochkin, Mohun Biswas, Richard Atterer, Maxim Perenesenko, - Daniel Egger, Jeff Johnson + Daniel Egger, Jeff Johnson, Nikitinskit Dmitriy, Georg Lippitsch Thanks! (and sorry if I forgot to mention someone) diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3 index 96f2d2b2c..56b763fe7 100644 --- a/docs/libcurl/curl_easy_setopt.3 +++ b/docs/libcurl/curl_easy_setopt.3 @@ -171,11 +171,32 @@ something special I/O-related needs to be done that the library can't do by itself. For now, rewinding the read data stream is the only action it can request. The rewinding of the read data stream may be necessary when doing a HTTP PUT or POST with a multi-pass authentication method. (Option added in -7.12.3) +7.12.3). + +Use \fICURLOPT_SEEKFUNCTION\fP instead to provide seeking! .IP CURLOPT_IOCTLDATA Pass a pointer that will be untouched by libcurl and passed as the 3rd argument in the ioctl callback set with \fICURLOPT_IOCTLFUNCTION\fP. (Option added in 7.12.3) +.IP CURLOPT_SEEKFUNCTION +Function pointer that should match the following prototype: \fIint +function(void *instream, curl_off_t offset, int origin);\fP This function gets +called by libcurl to seek to a certain position in the input stream and can be +used to fast forward a file in a resumed upload (instead of reading all +uploaded bytes with the normal read function/callback). It is also called to +rewind a stream when doing a HTTP PUT or POST with a multi-pass authentication +method. The function shall work like "fseek" or "lseek" and accepted SEEK_SET, +SEEK_CUR and SEEK_END as argument for origin, although (in 7.18.0) libcurl +only passes SEEK_SET. The callback must return 0 on success as returning +non-zero will cause the upload operation to fail. + +If you forward the input arguments directly to "fseek" or "lseek", note that +the data type for \fIoffset\fP is not the same as defined for curl_off_t on +many systems! (Option added in 7.18.0) +.IP CURLOPT_SEEKDATA +Data pointer to pass to the file read function. If you use the +\fICURLOPT_SEEKFUNCTION\fP option, this is the pointer you'll get as input. If +you don't specify a seek callback, NULL is passed. (Option added in 7.18.0) .IP CURLOPT_SOCKOPTFUNCTION Function pointer that should match the \fIcurl_sockopt_callback\fP prototype found in \fI<curl/curl.h>\fP. This function gets called by libcurl after the diff --git a/include/curl/curl.h b/include/curl/curl.h index b6dcc4a85..f303d426c 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -244,6 +244,10 @@ typedef size_t (*curl_write_callback)(char *buffer, /* This is a return code for the read callback that, when returned, will signal libcurl to pause sending data on the current transfer. */ #define CURL_READFUNC_PAUSE 0x10000001 +typedef int (*curl_seek_callback)(void *instream, + curl_off_t offset, + int origin); /* 'whence' */ + typedef size_t (*curl_read_callback)(char *buffer, size_t size, size_t nitems, @@ -1180,6 +1184,10 @@ typedef enum { /* set transfer mode (;type=<a|i>) when doing FTP via an HTTP proxy */ CINIT(PROXY_TRANSFER_MODE, LONG, 166), + /* Callback function for seeking in the input stream */ + CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167), + CINIT(SEEKDATA, OBJECTPOINT, 168), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; @@ -1525,7 +1525,6 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn, struct FTP *ftp = conn->data->state.proto.ftp; struct SessionHandle *data = conn->data; struct ftp_conn *ftpc = &conn->proto.ftpc; - curl_off_t passed=0; if((data->state.resume_from && !sizechecked) || ((data->state.resume_from > 0) && sizechecked)) { @@ -1552,31 +1551,39 @@ static CURLcode ftp_state_ul_setup(struct connectdata *conn, /* enable append */ data->set.ftp_append = TRUE; - /* Let's read off the proper amount of bytes from the input. If we knew it - was a proper file we could've just fseek()ed but we only have a stream - here */ + /* Let's read off the proper amount of bytes from the input. */ + if(conn->seek_func) { + curl_off_t readthisamountnow = data->state.resume_from; - /* TODO: allow the ioctlfunction to provide a fast forward function that - can be used here and use this method only as a fallback! */ - do { - curl_off_t readthisamountnow = (data->state.resume_from - passed); - curl_off_t actuallyread; - - if(readthisamountnow > BUFSIZE) - readthisamountnow = BUFSIZE; - - actuallyread = (curl_off_t) - conn->fread_func(data->state.buffer, 1, (size_t)readthisamountnow, - conn->fread_in); - - passed += actuallyread; - if((actuallyread <= 0) || (actuallyread > readthisamountnow)) { - /* this checks for greater-than only to make sure that the - CURL_READFUNC_ABORT return code still aborts */ - failf(data, "Failed to read data"); + if(conn->seek_func(conn->seek_client, + readthisamountnow, SEEK_SET) != 0) { + failf(data, "Could not seek stream"); return CURLE_FTP_COULDNT_USE_REST; } - } while(passed < data->state.resume_from); + } + + else { + curl_off_t passed=0; + do { + curl_off_t readthisamountnow = (data->state.resume_from - passed); + curl_off_t actuallyread; + + if(readthisamountnow > BUFSIZE) + readthisamountnow = BUFSIZE; + + actuallyread = (curl_off_t) + conn->fread_func(data->state.buffer, 1, (size_t)readthisamountnow, + conn->fread_in); + + passed += actuallyread; + if((actuallyread <= 0) || (actuallyread > readthisamountnow)) { + /* this checks for greater-than only to make sure that the + CURL_READFUNC_ABORT return code still aborts */ + failf(data, "Failed to read data"); + return CURLE_FTP_COULDNT_USE_REST; + } + } while(passed < data->state.resume_from); + } /* now, decrease the size of the read */ if(data->set.infilesize>0) { diff --git a/lib/http.c b/lib/http.c index e41a8f750..fd58c06cc 100644 --- a/lib/http.c +++ b/lib/http.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2007, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2008, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -1793,6 +1793,8 @@ CURLcode Curl_http_done(struct connectdata *conn, /* set the proper values (possibly modified on POST) */ conn->fread_func = data->set.fread_func; /* restore */ conn->fread_in = data->set.in; /* restore */ + conn->seek_func = data->set.seek_func; /* restore */ + conn->seek_client = data->set.seek_client; /* restore */ if(http == NULL) return CURLE_OK; @@ -2186,30 +2188,41 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) if(data->state.resume_from && !data->state.this_is_a_follow) { /* do we still game? */ - curl_off_t passed=0; /* Now, let's read off the proper amount of bytes from the - input. If we knew it was a proper file we could've just - fseek()ed but we only have a stream here */ - do { - size_t readthisamountnow = (size_t)(data->state.resume_from - passed); - size_t actuallyread; - - if(readthisamountnow > BUFSIZE) - readthisamountnow = BUFSIZE; - - actuallyread = - data->set.fread_func(data->state.buffer, 1, (size_t)readthisamountnow, - data->set.in); - - passed += actuallyread; - if(actuallyread != readthisamountnow) { - failf(data, "Could only read %" FORMAT_OFF_T - " bytes from the input", - passed); + input. */ + if(conn->seek_func) { + curl_off_t readthisamountnow = data->state.resume_from; + + if(conn->seek_func(conn->seek_client, + readthisamountnow, SEEK_SET) != 0) { + failf(data, "Could not seek stream"); return CURLE_READ_ERROR; } - } while(passed != data->state.resume_from); /* loop until done */ + } + else { + curl_off_t passed=0; + + do { + size_t readthisamountnow = (size_t)(data->state.resume_from - passed); + size_t actuallyread; + + if(readthisamountnow > BUFSIZE) + readthisamountnow = BUFSIZE; + + actuallyread = data->set.fread_func(data->state.buffer, 1, + (size_t)readthisamountnow, + data->set.in); + + passed += actuallyread; + if(actuallyread != readthisamountnow) { + failf(data, "Could only read %" FORMAT_OFF_T + " bytes from the input", + passed); + return CURLE_READ_ERROR; + } + } while(passed != data->state.resume_from); /* loop until done */ + } /* now, decrease the size of the read */ if(data->set.infilesize>0) { diff --git a/lib/transfer.c b/lib/transfer.c index 77146dd6e..512eceff0 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -245,11 +245,20 @@ CURLcode Curl_readrewind(struct connectdata *conn) (data->set.httpreq == HTTPREQ_POST_FORM)) ; /* do nothing */ else { - if(data->set.ioctl_func) { + if(data->set.seek_func) { + int err; + + err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET); + if(err) { + failf(data, "seek callback returned error %d\n", (int)err); + return CURLE_SEND_FAIL_REWIND; + } + } + else if(data->set.ioctl_func) { curlioerr err; - err = (data->set.ioctl_func) (data, CURLIOCMD_RESTARTREAD, - data->set.ioctl_client); + err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD, + data->set.ioctl_client); infof(data, "the ioctl callback returned %d\n", (int)err); if(err) { @@ -685,6 +685,10 @@ CURLcode Curl_open(struct SessionHandle **curl) /* use fread as default function to read input */ data->set.fread_func = (curl_read_callback)fread; + /* don't use a seek function by default */ + data->set.seek_func = ZERO_NULL; + data->set.seek_client = ZERO_NULL; + /* conversion callbacks for non-ASCII hosts */ data->set.convfromnetwork = ZERO_NULL; data->set.convtonetwork = ZERO_NULL; @@ -1627,6 +1631,18 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, /* When set to NULL, reset to our internal default function */ data->set.fread_func = (curl_read_callback)fread; break; + case CURLOPT_SEEKFUNCTION: + /* + * Seek callback. Might be NULL. + */ + data->set.seek_func = va_arg(param, curl_seek_callback); + break; + case CURLOPT_SEEKDATA: + /* + * Seek control callback. Might be NULL. + */ + data->set.seek_client = va_arg(param, void *); + break; case CURLOPT_CONV_FROM_NETWORK_FUNCTION: /* * "Convert from network encoding" callback @@ -4038,6 +4054,8 @@ static CURLcode CreateConnection(struct SessionHandle *data, * the persistent connection stuff */ conn->fread_func = data->set.fread_func; conn->fread_in = data->set.in; + conn->seek_func = data->set.seek_func; + conn->seek_client = data->set.seek_client; if((conn->protocol&PROT_HTTP) && data->set.upload && diff --git a/lib/urldata.h b/lib/urldata.h index 976bc3771..e918cd61d 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -961,6 +961,9 @@ struct connectdata { size_t buf_len; /* Length of the buffer?? */ + curl_seek_callback seek_func; /* function that seeks the input */ + void *seek_client; /* pointer to pass to the seek() above */ + /*************** Request - specific items ************/ /* previously this was in the urldata struct */ @@ -1324,6 +1327,7 @@ struct UserDefined { bool free_referer; /* set TRUE if 'referer' points to a string we allocated */ void *postfields; /* if POST, set the fields' values here */ + curl_seek_callback seek_func; /* function that seeks the input */ curl_off_t postfieldsize; /* if POST, this might have a size to use instead of strlen(), and then the data *may* be binary (contain zero bytes) */ @@ -1342,6 +1346,7 @@ struct UserDefined { the address and opening the socket */ void* opensocket_client; + void *seek_client; /* pointer to pass to the seek callback */ /* the 3 curl_conv_callback functions below are used on non-ASCII hosts */ /* function to convert from the network encoding: */ curl_conv_callback convfromnetwork; diff --git a/src/main.c b/src/main.c index c86bf2fe7..2b6cf4ab8 100644 --- a/src/main.c +++ b/src/main.c @@ -3020,6 +3020,7 @@ struct InStruct { struct Configurable *config; }; +#if 1 static curlioerr my_ioctl(CURL *handle, curliocmd cmd, void *userp) { struct InStruct *in=(struct InStruct *)userp; @@ -3040,6 +3041,24 @@ static curlioerr my_ioctl(CURL *handle, curliocmd cmd, void *userp) } return CURLIOE_OK; } +#else +static int my_seek(void *stream, curl_off_t offset, int whence) +{ + struct InStruct *in=(struct InStruct *)stream; + + /* We can't use fseek() here since it can't do 64bit seeks on Windows and + possibly elsewhere. We need to switch to the lseek family of tricks. For + that to work, we need to switch from fread() to plain read() etc */ + + if(-1 == fseek(in->stream, (off_t)offset, whence)) + /* couldn't rewind, the reason is in errno but errno is just not + portable enough and we don't actually care that much why we failed. */ + return 1; + + return 0; +} + +#endif static size_t my_fread(void *buffer, size_t sz, size_t nmemb, void *userp) { @@ -4270,10 +4289,17 @@ operate(struct Configurable *config, int argc, argv_item_t argv[]) /* what call to read */ my_setopt(curl, CURLOPT_READFUNCTION, my_fread); +#if 1 /* the ioctl function is at this point only used to rewind files that are posted when using NTLM etc */ my_setopt(curl, CURLOPT_IOCTLDATA, &input); my_setopt(curl, CURLOPT_IOCTLFUNCTION, my_ioctl); +#else + /* in 7.18.0, the SEEKFUNCTION/DATA pair is taking over what IOCTL* + previously provided for seeking */ + my_setopt(curl, CURLOPT_SEEKDATA, &input); + my_setopt(curl, CURLOPT_SEEKFUNCTION, my_seek); +#endif if(config->recvpersecond) /* tell libcurl to use a smaller sized buffer as it allows us to |