diff options
37 files changed, 2210 insertions, 279 deletions
diff --git a/docs/libcurl/curl_multi_setopt.3 b/docs/libcurl/curl_multi_setopt.3 index 9e456a093..99984cf49 100644 --- a/docs/libcurl/curl_multi_setopt.3 +++ b/docs/libcurl/curl_multi_setopt.3 @@ -95,6 +95,112 @@ This option is for the multi handle's use only, when using the easy interface you should instead use the \fICURLOPT_MAXCONNECTS\fP option. (Added in 7.16.3) +.IP CURLMOPT_MAX_HOST_CONNECTIONS +Pass a long. The set number will be used as the maximum amount of +simultaneously open connections to a single host. For each new session to +a host, libcurl will open a new connection up to the limit set by +CURLMOPT_MAX_HOST_CONNECTIONS. When the limit is reached, the sessions will +be pending until there are available connections. If CURLMOPT_PIPELINING is +1, libcurl will try to pipeline if the host is capable of it. + +The default value is 0, which means that there is no limit. +However, for backwards compatibility, setting it to 0 when CURLMOPT_PIPELINING +is 1 will not be treated as unlimited. Instead it will open only 1 connection +and try to pipeline on it. + +(Added in 7.30.0) +.IP CURLMOPT_MAX_PIPELINE_LENGTH +Pass a long. The set number will be used as the maximum amount of requests +in a pipelined connection. When this limit is reached, libcurl will use another +connection to the same host (see CURLMOPT_MAX_HOST_CONNECTIONS), or queue the +requests until one of the pipelines to the host is ready to accept a request. +Thus, the total number of requests in-flight is CURLMOPT_MAX_HOST_CONNECTIONS * +CURLMOPT_MAX_PIPELINE_LENGTH. +The default value is 5. + +(Added in 7.30.0) +.IP CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE +Pass a long. If a pipelined connection is currently processing a request +with a Content-Length larger than CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE, that +connection will not be considered for additional requests, even if it is +shorter than CURLMOPT_MAX_PIPELINE_LENGTH. +The default value is 0, which means that the penalization is inactive. + +(Added in 7.30.0) +.IP CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE +Pass a long. If a pipelined connection is currently processing a +chunked (Transfer-encoding: chunked) request with a current chunk length +larger than CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE, that connection will not be +considered for additional requests, even if it is shorter than +CURLMOPT_MAX_PIPELINE_LENGTH. +The default value is 0, which means that the penalization is inactive. + +(Added in 7.30.0) +.IP CURLMOPT_PIPELINING_SITE_BL +Pass an array of char *, ending with NULL. This is a list of sites that are +blacklisted from pipelining, i.e sites that are known to not support HTTP +pipelining. The array is copied by libcurl. + +The default value is NULL, which means that there is no blacklist. + +Pass a NULL pointer to clear the blacklist. + +Example: + +.nf + site_blacklist[] = + { + "www.haxx.se", + "www.example.com:1234", + NULL + }; + + curl_multi_setopt(m, CURLMOPT_PIPELINE_SITE_BL, site_blacklist); +.fi + +(Added in 7.30.0) +.IP CURLMOPT_PIPELINING_SERVER_BL +Pass an array of char *, ending with NULL. This is a list of server types +prefixes (in the Server: HTTP header) that are blacklisted from pipelining, +i.e server types that are known to not support HTTP pipelining. The array is +copied by libcurl. + +Note that the comparison matches if the Server: header begins with the string +in the blacklist, i.e "Server: Ninja 1.2.3" and "Server: Ninja 1.4.0" can +both be blacklisted by having "Ninja" in the backlist. + +The default value is NULL, which means that there is no blacklist. + +Pass a NULL pointer to clear the blacklist. + +Example: + +.nf + server_blacklist[] = + { + "Microsoft-IIS/6.0", + "nginx/0.8.54", + NULL + }; + + curl_multi_setopt(m, CURLMOPT_PIPELINE_SERVER_BL, server_blacklist); +.fi + +(Added in 7.30.0) +.IP CURLMOPT_MAX_TOTAL_CONNECTIONS +Pass a long. The set number will be used as the maximum amount of +simultaneously open connections in total. For each new session, libcurl +will open a new connection up to the limit set by +CURLMOPT_MAX_TOTAL_CONNECTIONS. When the limit is reached, the sessions will +be pending until there are available connections. If CURLMOPT_PIPELINING is +1, libcurl will try to pipeline if the host is capable of it. + +The default value is 0, which means that there is no limit. +However, for backwards compatibility, setting it to 0 when CURLMOPT_PIPELINING +is 1 will not be treated as unlimited. Instead it will open only 1 connection +and try to pipeline on it. + +(Added in 7.30.0) .SH RETURNS The standard CURLMcode for multi interface error codes. Note that it returns a CURLM_UNKNOWN_OPTION if you try setting an option that this version of libcurl diff --git a/docs/libcurl/libcurl-errors.3 b/docs/libcurl/libcurl-errors.3 index beee3971f..7b6823735 100644 --- a/docs/libcurl/libcurl-errors.3 +++ b/docs/libcurl/libcurl-errors.3 @@ -240,6 +240,9 @@ Mismatch of RTSP Session Identifiers. Unable to parse FTP file list (during FTP wildcard downloading). .IP "CURLE_CHUNK_FAILED (88)" Chunk callback reported error. +.IP "CURLE_NO_CONNECTION_AVAILABLE (89)" +(For internal use only, will never be returned by libcurl) No connection +available, the session will be queued. (added in 7.30.0) .IP "CURLE_OBSOLETE*" These error codes will never be returned. They were used in an old libcurl version and are currently unused. diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index 37b5e277d..5ed3f8477 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -85,6 +85,7 @@ CURLE_LDAP_SEARCH_FAILED 7.1 CURLE_LIBRARY_NOT_FOUND 7.1 7.17.0 CURLE_LOGIN_DENIED 7.13.1 CURLE_MALFORMAT_USER 7.1 7.17.0 +CURLE_NO_CONNECTION_AVAILABLE 7.30.0 CURLE_NOT_BUILT_IN 7.21.5 CURLE_OK 7.1 CURLE_OPERATION_TIMEDOUT 7.10.2 @@ -267,8 +268,15 @@ CURLKHTYPE_DSS 7.19.6 CURLKHTYPE_RSA 7.19.6 CURLKHTYPE_RSA1 7.19.6 CURLKHTYPE_UNKNOWN 7.19.6 +CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE 7.30.0 +CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE 7.30.0 +CURLMOPT_MAX_HOST_CONNECTIONS 7.30.0 +CURLMOPT_MAX_PIPELINE_LENGTH 7.30.0 +CURLMOPT_MAX_TOTAL_CONNECTIONS 7.30.0 CURLMOPT_MAXCONNECTS 7.16.3 CURLMOPT_PIPELINING 7.16.0 +CURLMOPT_PIPELINING_SERVER_BL 7.30.0 +CURLMOPT_PIPELINING_SITE_BL 7.30.0 CURLMOPT_SOCKETDATA 7.15.4 CURLMOPT_SOCKETFUNCTION 7.15.4 CURLMOPT_TIMERDATA 7.16.0 diff --git a/include/curl/curl.h b/include/curl/curl.h index 0b2b7ea44..c7a052d1e 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -507,6 +507,8 @@ typedef enum { CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ + CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the + session will be queued */ CURL_LAST /* never use! */ } CURLcode; diff --git a/include/curl/multi.h b/include/curl/multi.h index 6dcd2bac4..a5eb3c643 100644 --- a/include/curl/multi.h +++ b/include/curl/multi.h @@ -338,6 +338,31 @@ typedef enum { /* maximum number of entries in the connection cache */ CINIT(MAXCONNECTS, LONG, 6), + /* maximum number of (pipelining) connections to one host */ + CINIT(MAX_HOST_CONNECTIONS, LONG, 7), + + /* maximum number of requests in a pipeline */ + CINIT(MAX_PIPELINE_LENGTH, LONG, 8), + + /* a connection with a content-length longer than this + will not be considered for pipelining */ + CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), + + /* a connection with a chunk length longer than this + will not be considered for pipelining */ + CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), + + /* a list of site names(+port) that are blacklisted from + pipelining */ + CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), + + /* a list of server types that are blacklisted from + pipelining */ + CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), + + /* maximum number of open connections in total */ + CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), + CURLMOPT_LASTENTRY /* the last unused */ } CURLMoption; diff --git a/lib/Makefile.inc b/lib/Makefile.inc index db0597365..f76e1ec83 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -25,7 +25,7 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ http_proxy.c non-ascii.c asyn-ares.c asyn-thread.c curl_gssapi.c \ curl_ntlm.c curl_ntlm_wb.c curl_ntlm_core.c curl_ntlm_msgs.c \ curl_sasl.c curl_schannel.c curl_multibyte.c curl_darwinssl.c \ - hostcheck.c bundles.c conncache.c + hostcheck.c bundles.c conncache.c pipeline.c HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h \ @@ -44,4 +44,4 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ asyn.h curl_ntlm.h curl_gssapi.h curl_ntlm_wb.h curl_ntlm_core.h \ curl_ntlm_msgs.h curl_sasl.h curl_schannel.h curl_multibyte.h \ curl_darwinssl.h hostcheck.h bundles.h conncache.h curl_setup_once.h \ - multihandle.h setup-vms.h + multihandle.h setup-vms.h pipeline.h diff --git a/lib/README.pipelining b/lib/README.pipelining index c7b462248..e5bf6ec33 100644 --- a/lib/README.pipelining +++ b/lib/README.pipelining @@ -42,10 +42,3 @@ Details still resolve the second one properly to make sure that they actually _can_ be considered for pipelining. Also, asking for explicit pipelining on handle X may be tricky when handle X get a closed connection. - -- We need options to control max pipeline length, and probably how to behave - if we reach that limit. As was discussed on the list, it can probably be - made very complicated, so perhaps we can think of a way to pass all - variables involved to a callback and let the application decide how to act - in specific situations. Either way, these fancy options are only interesting - to work on when everything is working and we have working apps to test with. diff --git a/lib/hash.h b/lib/hash.h index 99a627405..aa935d4eb 100644 --- a/lib/hash.h +++ b/lib/hash.h @@ -104,4 +104,3 @@ void Curl_hash_print(struct curl_hash *h, #endif /* HEADER_CURL_HASH_H */ - diff --git a/lib/http.c b/lib/http.c index daaafe317..0ba11133f 100644 --- a/lib/http.c +++ b/lib/http.c @@ -73,6 +73,8 @@ #include "http_proxy.h" #include "warnless.h" #include "non-ascii.h" +#include "bundles.h" +#include "pipeline.h" #define _MPRINTF_REPLACE /* use our functions only */ #include <curl/mprintf.h> @@ -3148,13 +3150,19 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data, } else if(conn->httpversion >= 11 && !conn->bits.close) { + struct connectbundle *cb_ptr; /* If HTTP version is >= 1.1 and connection is persistent server supports pipelining. */ DEBUGF(infof(data, "HTTP 1.1 or later with persistent connection, " "pipelining supported\n")); - conn->server_supports_pipelining = TRUE; + /* Activate pipelining if needed */ + cb_ptr = conn->bundle; + if(cb_ptr) { + if(!Curl_pipeline_site_blacklisted(data, conn)) + cb_ptr->server_supports_pipelining = TRUE; + } } switch(k->httpcode) { @@ -3231,6 +3239,16 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data, data->info.contenttype = contenttype; } } + else if(checkprefix("Server:", k->p)) { + char *server_name = copy_header_value(k->p); + + /* Turn off pipelining if the server version is blacklisted */ + if(conn->bundle && conn->bundle->server_supports_pipelining) { + if(Curl_pipeline_server_blacklisted(data, server_name)) + conn->bundle->server_supports_pipelining = FALSE; + } + Curl_safefree(server_name); + } else if((conn->httpversion == 10) && conn->bits.httpproxy && Curl_compareheader(k->p, diff --git a/lib/multi.c b/lib/multi.c index a369d0361..3e2583a21 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -40,6 +40,7 @@ #include "conncache.h" #include "bundles.h" #include "multihandle.h" +#include "pipeline.h" #define _MPRINTF_REPLACE /* use our functions only */ #include <curl/mprintf.h> @@ -69,13 +70,6 @@ static void singlesocket(struct Curl_multi *multi, struct Curl_one_easy *easy); static int update_timer(struct Curl_multi *multi); -static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle, - struct connectdata *conn); -static int checkPendPipeline(struct connectdata *conn); -static void moveHandleFromSendToRecvPipeline(struct SessionHandle *handle, - struct connectdata *conn); -static void moveHandleFromRecvToDonePipeline(struct SessionHandle *handle, - struct connectdata *conn); static bool isHandleAtHead(struct SessionHandle *handle, struct curl_llist *pipeline); static CURLMcode add_next_timeout(struct timeval now, @@ -85,6 +79,7 @@ static CURLMcode add_next_timeout(struct timeval now, #ifdef DEBUGBUILD static const char * const statename[]={ "INIT", + "CONNECT_PEND", "CONNECT", "WAITRESOLVE", "WAITCONNECT", @@ -125,9 +120,9 @@ static void mstate(struct Curl_one_easy *easy, CURLMstate state easy->state = state; #ifdef DEBUGBUILD - if(easy->easy_conn) { - if(easy->state > CURLM_STATE_CONNECT && - easy->state < CURLM_STATE_COMPLETED) + if(easy->state >= CURLM_STATE_CONNECT_PEND && + easy->state < CURLM_STATE_COMPLETED) { + if(easy->easy_conn) connection_id = easy->easy_conn->connection_id; infof(easy->easy_handle, @@ -314,6 +309,7 @@ CURLM *curl_multi_init(void) multi->easy.next = &multi->easy; multi->easy.prev = &multi->easy; + multi->max_pipeline_length = 5; return (CURLM *) multi; error: @@ -580,7 +576,7 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle, /* as this was using a shared connection cache we clear the pointer to that since we're not part of that multi handle anymore */ - easy->easy_handle->state.conn_cache = NULL; + easy->easy_handle->state.conn_cache = NULL; /* change state without using multistate(), only to make singlesocket() do what we want */ @@ -638,9 +634,9 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle, return CURLM_BAD_EASY_HANDLE; /* twasn't found */ } -bool Curl_multi_canPipeline(const struct Curl_multi* multi) +bool Curl_multi_pipeline_enabled(const struct Curl_multi* multi) { - return multi->pipelining_enabled; + return multi && multi->pipelining_enabled; } void Curl_multi_handlePipeBreak(struct SessionHandle *data) @@ -1007,16 +1003,27 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } break; + case CURLM_STATE_CONNECT_PEND: + /* We will stay here until there is a connection available. Then + we try again in the CURLM_STATE_CONNECT state. */ + break; + case CURLM_STATE_CONNECT: - /* Connect. We get a connection identifier filled in. */ + /* Connect. We want to get a connection identifier filled in. */ Curl_pgrsTime(data, TIMER_STARTSINGLE); easy->result = Curl_connect(data, &easy->easy_conn, &async, &protocol_connect); + if(CURLE_NO_CONNECTION_AVAILABLE == easy->result) { + /* There was no connection available. We will go to the pending + state and wait for an available connection. */ + multistate(easy, CURLM_STATE_CONNECT_PEND); + easy->result = CURLM_OK; + break; + } if(CURLE_OK == easy->result) { /* Add this handle to the send or pend pipeline */ - easy->result = addHandleToSendOrPendPipeline(data, - easy->easy_conn); + easy->result = Curl_add_handle_to_pipeline(data, easy->easy_conn); if(CURLE_OK != easy->result) disconnect_conn = TRUE; else { @@ -1357,9 +1364,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, case CURLM_STATE_DO_DONE: /* Move ourselves from the send to recv pipeline */ - moveHandleFromSendToRecvPipeline(data, easy->easy_conn); + Curl_move_handle_from_send_to_recv_pipe(data, easy->easy_conn); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); multistate(easy, CURLM_STATE_WAITPERFORM); result = CURLM_CALL_MULTI_PERFORM; break; @@ -1491,15 +1498,14 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, Curl_posttransfer(data); /* we're no longer receiving */ - moveHandleFromRecvToDonePipeline(data, - easy->easy_conn); + Curl_removeHandleFromPipeline(data, easy->easy_conn->recv_pipe); /* expire the new receiving pipeline head */ if(easy->easy_conn->recv_pipe->head) Curl_expire(easy->easy_conn->recv_pipe->head->ptr, 1); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); /* When we follow redirects or is set to retry the connection, we must to go back to the CONNECT state */ @@ -1554,14 +1560,11 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, case CURLM_STATE_DONE: if(easy->easy_conn) { - /* Remove ourselves from the receive and done pipelines. Handle - should be on one of these lists, depending upon how we got here. */ + /* Remove ourselves from the receive pipeline, if we are there. */ Curl_removeHandleFromPipeline(data, easy->easy_conn->recv_pipe); - Curl_removeHandleFromPipeline(data, - easy->easy_conn->done_pipe); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); if(easy->easy_conn->bits.stream_was_rewound) { /* This request read past its response boundary so we quickly let @@ -1638,10 +1641,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, easy->easy_conn->send_pipe); Curl_removeHandleFromPipeline(data, easy->easy_conn->recv_pipe); - Curl_removeHandleFromPipeline(data, - easy->easy_conn->done_pipe); /* Check if we can move pending requests to send pipe */ - checkPendPipeline(easy->easy_conn); + Curl_multi_process_pending_handles(multi); if(disconnect_conn) { /* disconnect properly */ @@ -1789,8 +1790,8 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle) Curl_hostcache_clean(multi->closure_handle); Curl_close(multi->closure_handle); + multi->closure_handle = NULL; } - multi->closure_handle = NULL; Curl_hash_destroy(multi->sockhash); multi->sockhash = NULL; @@ -1825,6 +1826,10 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle) Curl_hash_destroy(multi->hostcache); multi->hostcache = NULL; + /* Free the blacklists by setting them to NULL */ + Curl_pipeline_set_site_blacklist(NULL, &multi->pipelining_site_bl); + Curl_pipeline_set_server_blacklist(NULL, &multi->pipelining_server_bl); + free(multi); return CURLM_OK; @@ -2242,6 +2247,29 @@ CURLMcode curl_multi_setopt(CURLM *multi_handle, case CURLMOPT_MAXCONNECTS: multi->maxconnects = va_arg(param, long); break; + case CURLMOPT_MAX_HOST_CONNECTIONS: + multi->max_host_connections = va_arg(param, long); + break; + case CURLMOPT_MAX_PIPELINE_LENGTH: + multi->max_pipeline_length = va_arg(param, long); + break; + case CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE: + multi->content_length_penalty_size = va_arg(param, long); + break; + case CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE: + multi->chunk_length_penalty_size = va_arg(param, long); + break; + case CURLMOPT_PIPELINING_SITE_BL: + res = Curl_pipeline_set_site_blacklist(va_arg(param, char **), + &multi->pipelining_site_bl); + break; + case CURLMOPT_PIPELINING_SERVER_BL: + res = Curl_pipeline_set_server_blacklist(va_arg(param, char **), + &multi->pipelining_server_bl); + break; + case CURLMOPT_MAX_TOTAL_CONNECTIONS: + multi->max_total_connections = va_arg(param, long); + break; default: res = CURLM_UNKNOWN_OPTION; break; @@ -2366,131 +2394,12 @@ static int update_timer(struct Curl_multi *multi) return multi->timer_cb((CURLM*)multi, timeout_ms, multi->timer_userp); } -static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle, +void Curl_multi_set_easy_connection(struct SessionHandle *handle, struct connectdata *conn) { - size_t pipeLen = conn->send_pipe->size + conn->recv_pipe->size; - struct curl_llist_element *sendhead = conn->send_pipe->head; - struct curl_llist *pipeline; - CURLcode rc; - - if(!Curl_isPipeliningEnabled(handle) || - pipeLen == 0) - pipeline = conn->send_pipe; - else { - if(conn->server_supports_pipelining && - pipeLen < MAX_PIPELINE_LENGTH) - pipeline = conn->send_pipe; - else - pipeline = conn->pend_pipe; - } - - rc = Curl_addHandleToPipeline(handle, pipeline); - - if(pipeline == conn->send_pipe && sendhead != conn->send_pipe->head) { - /* this is a new one as head, expire it */ - conn->writechannel_inuse = FALSE; /* not in use yet */ -#ifdef DEBUGBUILD - infof(conn->data, "%p is at send pipe head!\n", - conn->send_pipe->head->ptr); -#endif - Curl_expire(conn->send_pipe->head->ptr, 1); - } - - return rc; -} - -static int checkPendPipeline(struct connectdata *conn) -{ - int result = 0; - struct curl_llist_element *sendhead = conn->send_pipe->head; - - size_t pipeLen = conn->send_pipe->size + conn->recv_pipe->size; - if(conn->server_supports_pipelining || pipeLen == 0) { - struct curl_llist_element *curr = conn->pend_pipe->head; - const size_t maxPipeLen = - conn->server_supports_pipelining ? MAX_PIPELINE_LENGTH : 1; - - while(pipeLen < maxPipeLen && curr) { - Curl_llist_move(conn->pend_pipe, curr, - conn->send_pipe, conn->send_pipe->tail); - Curl_pgrsTime(curr->ptr, TIMER_PRETRANSFER); - ++result; /* count how many handles we moved */ - curr = conn->pend_pipe->head; - ++pipeLen; - } - } - - if(result) { - conn->now = Curl_tvnow(); - /* something moved, check for a new send pipeline leader */ - if(sendhead != conn->send_pipe->head) { - /* this is a new one as head, expire it */ - conn->writechannel_inuse = FALSE; /* not in use yet */ -#ifdef DEBUGBUILD - infof(conn->data, "%p is at send pipe head!\n", - conn->send_pipe->head->ptr); -#endif - Curl_expire(conn->send_pipe->head->ptr, 1); - } - } - - return result; -} - -/* Move this transfer from the sending list to the receiving list. - - Pay special attention to the new sending list "leader" as it needs to get - checked to update what sockets it acts on. - -*/ -static void moveHandleFromSendToRecvPipeline(struct SessionHandle *handle, - struct connectdata *conn) -{ - struct curl_llist_element *curr; - - curr = conn->send_pipe->head; - while(curr) { - if(curr->ptr == handle) { - Curl_llist_move(conn->send_pipe, curr, - conn->recv_pipe, conn->recv_pipe->tail); - - if(conn->send_pipe->head) { - /* Since there's a new easy handle at the start of the send pipeline, - set its timeout value to 1ms to make it trigger instantly */ - conn->writechannel_inuse = FALSE; /* not used now */ -#ifdef DEBUGBUILD - infof(conn->data, "%p is at send pipe head B!\n", - conn->send_pipe->head->ptr); -#endif - Curl_expire(conn->send_pipe->head->ptr, 1); - } - - /* The receiver's list is not really interesting here since either this - handle is now first in the list and we'll deal with it soon, or - another handle is already first and thus is already taken care of */ - - break; /* we're done! */ - } - curr = curr->next; - } + handle->set.one_easy->easy_conn = conn; } -static void moveHandleFromRecvToDonePipeline(struct SessionHandle *handle, - struct connectdata *conn) -{ - struct curl_llist_element *curr; - - curr = conn->recv_pipe->head; - while(curr) { - if(curr->ptr == handle) { - Curl_llist_move(conn->recv_pipe, curr, - conn->done_pipe, conn->done_pipe->tail); - break; - } - curr = curr->next; - } -} static bool isHandleAtHead(struct SessionHandle *handle, struct curl_llist *pipeline) { @@ -2670,6 +2579,56 @@ CURLMcode curl_multi_assign(CURLM *multi_handle, return CURLM_OK; } +size_t Curl_multi_max_host_connections(struct Curl_multi *multi) +{ + return multi ? multi->max_host_connections : 0; +} + +size_t Curl_multi_max_total_connections(struct Curl_multi *multi) +{ + return multi ? multi->max_total_connections : 0; +} + +size_t Curl_multi_max_pipeline_length(struct Curl_multi *multi) +{ + return multi ? multi->max_pipeline_length : 0; +} + +curl_off_t Curl_multi_content_length_penalty_size(struct Curl_multi *multi) +{ + return multi ? multi->content_length_penalty_size : 0; +} + +curl_off_t Curl_multi_chunk_length_penalty_size(struct Curl_multi *multi) +{ + return multi ? multi->chunk_length_penalty_size : 0; +} + +struct curl_llist *Curl_multi_pipelining_site_bl(struct Curl_multi *multi) +{ + return multi->pipelining_site_bl; +} + +struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi) +{ + return multi->pipelining_server_bl; +} + +void Curl_multi_process_pending_handles(struct Curl_multi *multi) +{ + struct Curl_one_easy *easy; + + easy=multi->easy.next; + while(easy != &multi->easy) { + if(easy->state == CURLM_STATE_CONNECT_PEND) { + multistate(easy, CURLM_STATE_CONNECT); + /* Make sure that the handle will be processed soonish. */ + Curl_expire(easy->easy_handle, 1); + } + easy = easy->next; /* operate on next handle */ + } +} + #ifdef DEBUGBUILD void Curl_multi_dump(const struct Curl_multi *multi_handle) { diff --git a/lib/multihandle.h b/lib/multihandle.h index 941835941..3fcd9c0e6 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -31,25 +31,26 @@ struct Curl_message { well! */ typedef enum { - CURLM_STATE_INIT, /* 0 - start in this state */ - CURLM_STATE_CONNECT, /* 1 - resolve/connect has been sent off */ - CURLM_STATE_WAITRESOLVE, /* 2 - awaiting the resolve to finalize */ - CURLM_STATE_WAITCONNECT, /* 3 - awaiting the connect to finalize */ - CURLM_STATE_WAITPROXYCONNECT, /* 4 - awaiting proxy CONNECT to finalize */ - CURLM_STATE_PROTOCONNECT, /* 5 - completing the protocol-specific connect - phase */ - CURLM_STATE_WAITDO, /* 6 - wait for our turn to send the request */ - CURLM_STATE_DO, /* 7 - start send off the request (part 1) */ - CURLM_STATE_DOING, /* 8 - sending off the request (part 1) */ - CURLM_STATE_DO_MORE, /* 9 - send off the request (part 2) */ - CURLM_STATE_DO_DONE, /* 10 - done sending off request */ - CURLM_STATE_WAITPERFORM, /* 11 - wait for our turn to read the response */ - CURLM_STATE_PERFORM, /* 12 - transfer data */ - CURLM_STATE_TOOFAST, /* 13 - wait because limit-rate exceeded */ - CURLM_STATE_DONE, /* 14 - post data transfer operation */ - CURLM_STATE_COMPLETED, /* 15 - operation complete */ - CURLM_STATE_MSGSENT, /* 16 - the operation complete message is sent */ - CURLM_STATE_LAST /* 17 - not a true state, never use this */ + CURLM_STATE_INIT, /* 0 - start in this state */ + CURLM_STATE_CONNECT_PEND, /* 1 - no connections, waiting for one */ + CURLM_STATE_CONNECT, /* 2 - resolve/connect has been sent off */ + CURLM_STATE_WAITRESOLVE, /* 3 - awaiting the resolve to finalize */ + CURLM_STATE_WAITCONNECT, /* 4 - awaiting the connect to finalize */ + CURLM_STATE_WAITPROXYCONNECT, /* 5 - awaiting proxy CONNECT to finalize */ + CURLM_STATE_PROTOCONNECT, /* 6 - completing the protocol-specific connect + phase */ + CURLM_STATE_WAITDO, /* 7 - wait for our turn to send the request */ + CURLM_STATE_DO, /* 8 - start send off the request (part 1) */ + CURLM_STATE_DOING, /* 9 - sending off the request (part 1) */ + CURLM_STATE_DO_MORE, /* 10 - send off the request (part 2) */ + CURLM_STATE_DO_DONE, /* 11 - done sending off request */ + CURLM_STATE_WAITPERFORM, /* 12 - wait for our turn to read the response */ + CURLM_STATE_PERFORM, /* 13 - transfer data */ + CURLM_STATE_TOOFAST, /* 14 - wait because limit-rate exceeded */ + CURLM_STATE_DONE, /* 15 - post data transfer operation */ + CURLM_STATE_COMPLETED, /* 16 - operation complete */ + CURLM_STATE_MSGSENT, /* 17 - the operation complete message is sent */ + CURLM_STATE_LAST /* 18 - not a true state, never use this */ } CURLMstate; /* we support N sockets per easy handle. Set the corresponding bit to what @@ -123,6 +124,30 @@ struct Curl_multi { long maxconnects; /* if >0, a fixed limit of the maximum number of entries we're allowed to grow the connection cache to */ + long max_host_connections; /* if >0, a fixed limit of the maximum number + of connections per host */ + + long max_total_connections; /* if >0, a fixed limit of the maximum number + of connections in total */ + + long max_pipeline_length; /* if >0, maximum number of requests in a + pipeline */ + + long content_length_penalty_size; /* a connection with a + content-length bigger than + this is not considered + for pipelining */ + + long chunk_length_penalty_size; /* a connection with a chunk length + bigger than this is not + considered for pipelining */ + + struct curl_llist *pipelining_site_bl; /* List of sites that are blacklisted + from pipelining */ + + struct curl_llist *pipelining_server_bl; /* List of server types that are + blacklisted from pipelining */ + /* timer callback and user data pointer for the *socket() API */ curl_multi_timer_callback timer_cb; void *timer_userp; diff --git a/lib/multiif.h b/lib/multiif.h index c84b6184c..0dcdec76e 100644 --- a/lib/multiif.h +++ b/lib/multiif.h @@ -27,7 +27,7 @@ */ void Curl_expire(struct SessionHandle *data, long milli); -bool Curl_multi_canPipeline(const struct Curl_multi* multi); +bool Curl_multi_pipeline_enabled(const struct Curl_multi* multi); void Curl_multi_handlePipeBreak(struct SessionHandle *data); /* the write bits start at bit 16 for the *getsock() bitmap */ @@ -50,5 +50,31 @@ void Curl_multi_handlePipeBreak(struct SessionHandle *data); void Curl_multi_dump(const struct Curl_multi *multi_handle); #endif -#endif /* HEADER_CURL_MULTIIF_H */ +/* Update the current connection of a One_Easy handle */ +void Curl_multi_set_easy_connection(struct SessionHandle *handle, + struct connectdata *conn); + +void Curl_multi_process_pending_handles(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_MAX_HOST_CONNECTIONS option */ +size_t Curl_multi_max_host_connections(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_MAX_PIPELINE_LENGTH option */ +size_t Curl_multi_max_pipeline_length(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE option */ +curl_off_t Curl_multi_content_length_penalty_size(struct Curl_multi *multi); +/* Return the value of the CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE option */ +curl_off_t Curl_multi_chunk_length_penalty_size(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_PIPELINING_SITE_BL option */ +struct curl_llist *Curl_multi_pipelining_site_bl(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_PIPELINING_SERVER_BL option */ +struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi); + +/* Return the value of the CURLMOPT_MAX_TOTAL_CONNECTIONS option */ +size_t Curl_multi_max_total_connections(struct Curl_multi *multi); + +#endif /* HEADER_CURL_MULTIIF_H */ diff --git a/lib/pipeline.c b/lib/pipeline.c new file mode 100644 index 000000000..7abc35fd0 --- /dev/null +++ b/lib/pipeline.c @@ -0,0 +1,366 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2013, Linus Nielsen Feltzing, <linus@haxx.se> + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "curl_setup.h" + +#include <curl/curl.h> + +#include "urldata.h" +#include "url.h" +#include "progress.h" +#include "multiif.h" +#include "pipeline.h" +#include "sendf.h" +#include "rawstr.h" +#include "bundles.h" + +#include "curl_memory.h" +/* The last #include file should be: */ +#include "memdebug.h" + +struct site_blacklist_entry { + char *hostname; + unsigned short port; +}; + +static void site_blacklist_llist_dtor(void *user, void *element) +{ + struct site_blacklist_entry *entry = element; + (void)user; + + Curl_safefree(entry->hostname); + Curl_safefree(entry); +} + +static void server_blacklist_llist_dtor(void *user, void *element) +{ + char *server_name = element; + (void)user; + + Curl_safefree(server_name); +} + +bool Curl_pipeline_penalized(struct SessionHandle *data, + struct connectdata *conn) +{ + if(data) { + bool penalized = FALSE; + curl_off_t penalty_size = + Curl_multi_content_length_penalty_size(data->multi); + curl_off_t chunk_penalty_size = + Curl_multi_chunk_length_penalty_size(data->multi); + curl_off_t recv_size = -2; /* Make it easy to spot in the log */ + + /* Find the head of the recv pipe, if any */ + if(conn->recv_pipe && conn->recv_pipe->head) { + struct SessionHandle *recv_handle = conn->recv_pipe->head->ptr; + + recv_size = recv_handle->req.size; + + if(penalty_size > 0 && recv_size > penalty_size) + penalized = TRUE; + } + + if(chunk_penalty_size > 0 && + (curl_off_t)conn->chunk.datasize > chunk_penalty_size) + penalized = TRUE; + + infof(data, "Conn: %d (%p) Receive pipe weight: (%d/%d), penalized: %d\n", + conn->connection_id, conn, recv_size, + conn->chunk.datasize, penalized); + return penalized; + } + return FALSE; +} + +/* Find the best connection in a bundle to use for the next request */ +struct connectdata * +Curl_bundle_find_best(struct SessionHandle *data, + struct connectbundle *cb_ptr) +{ + struct curl_llist_element *curr; + struct connectdata *conn; + struct connectdata *best_conn = NULL; + size_t pipe_len; + size_t best_pipe_len = 99; + + (void)data; + + curr = cb_ptr->conn_list->head; + while(curr) { + conn = curr->ptr; + pipe_len = conn->send_pipe->size + conn->recv_pipe->size; + + if(!Curl_pipeline_penalized(conn->data, conn) && + pipe_len < best_pipe_len) { + best_conn = conn; + best_pipe_len = pipe_len; + } + curr = curr->next; + } + + /* If we haven't found a connection, i.e all pipelines are penalized + or full, just pick one. The request will then be queued in + Curl_add_handle_to_pipeline(). */ + if(!best_conn) { + best_conn = cb_ptr->conn_list->head->ptr; + } + return best_conn; +} + +CURLcode Curl_add_handle_to_pipeline(struct SessionHandle *handle, + struct connectdata *conn) +{ + struct curl_llist_element *sendhead = conn->send_pipe->head; + struct curl_llist *pipeline; + CURLcode rc; + + pipeline = conn->send_pipe; + + infof(conn->data, "Adding handle: conn: %p\n", conn); + infof(conn->data, "Adding handle: send: %d\n", conn->send_pipe->size); + infof(conn->data, "Adding handle: recv: %d\n", conn->recv_pipe->size); + rc = Curl_addHandleToPipeline(handle, pipeline); + + if(pipeline == conn->send_pipe && sendhead != conn->send_pipe->head) { + /* this is a new one as head, expire it */ + conn->writechannel_inuse = FALSE; /* not in use yet */ +#ifdef DEBUGBUILD + infof(conn->data, "%p is at send pipe head!\n", + conn->send_pipe->head->ptr); +#endif + Curl_expire(conn->send_pipe->head->ptr, 1); + } + + print_pipeline(conn); + + return rc; +} + +/* Move this transfer from the sending list to the receiving list. + + Pay special attention to the new sending list "leader" as it needs to get + checked to update what sockets it acts on. + +*/ +void Curl_move_handle_from_send_to_recv_pipe(struct SessionHandle *handle, + struct connectdata *conn) +{ + struct curl_llist_element *curr; + + curr = conn->send_pipe->head; + while(curr) { + if(curr->ptr == handle) { + Curl_llist_move(conn->send_pipe, curr, + conn->recv_pipe, conn->recv_pipe->tail); + + if(conn->send_pipe->head) { + /* Since there's a new easy handle at the start of the send pipeline, + set its timeout value to 1ms to make it trigger instantly */ + conn->writechannel_inuse = FALSE; /* not used now */ +#ifdef DEBUGBUILD + infof(conn->data, "%p is at send pipe head B!\n", + conn->send_pipe->head->ptr); +#endif + Curl_expire(conn->send_pipe->head->ptr, 1); + } + + /* The receiver's list is not really interesting here since either this + handle is now first in the list and we'll deal with it soon, or + another handle is already first and thus is already taken care of */ + + break; /* we're done! */ + } + curr = curr->next; + } +} + +bool Curl_pipeline_site_blacklisted(struct SessionHandle *handle, + struct connectdata *conn) +{ + if(handle->multi) { + struct curl_llist *blacklist = + Curl_multi_pipelining_site_bl(handle->multi); + + if(blacklist) { + struct curl_llist_element *curr; + + curr = blacklist->head; + while(curr) { + struct site_blacklist_entry *site; + + site = curr->ptr; + if(Curl_raw_equal(site->hostname, conn->host.name) && + site->port == conn->remote_port) { + infof(handle, "Site %s:%d is pipeline blacklisted\n", + conn->host.name, conn->remote_port); + return TRUE; + } + curr = curr->next; + } + } + } + return FALSE; +} + +CURLMcode Curl_pipeline_set_site_blacklist(char **sites, + struct curl_llist **list_ptr) +{ + struct curl_llist *old_list = *list_ptr; + struct curl_llist *new_list = NULL; + + if(sites) { + new_list = Curl_llist_alloc((curl_llist_dtor) site_blacklist_llist_dtor); + if(!new_list) + return CURLM_OUT_OF_MEMORY; + + /* Parse the URLs and populate the list */ + while(*sites) { + char *hostname; + char *port; + struct site_blacklist_entry *entry; + + entry = malloc(sizeof(struct site_blacklist_entry)); + + hostname = strdup(*sites); + if(!hostname) + return CURLM_OUT_OF_MEMORY; + + port = strchr(hostname, ':'); + if(port) { + *port = '\0'; + port++; + entry->port = (unsigned short)strtol(port, NULL, 10); + } + else { + /* Default port number for HTTP */ + entry->port = 80; + } + + entry->hostname = hostname; + + if(!Curl_llist_insert_next(new_list, new_list->tail, entry)) + return CURLM_OUT_OF_MEMORY; + + sites++; + } + } + + /* Free the old list */ + if(old_list) { + Curl_llist_destroy(old_list, NULL); + } + + /* This might be NULL if sites == NULL, i.e the blacklist is cleared */ + *list_ptr = new_list; + + return CURLM_OK; +} + +bool Curl_pipeline_server_blacklisted(struct SessionHandle *handle, + char *server_name) +{ + if(handle->multi) { + struct curl_llist *blacklist = + Curl_multi_pipelining_server_bl(handle->multi); + + if(blacklist) { + struct curl_llist_element *curr; + + curr = blacklist->head; + while(curr) { + char *bl_server_name; + + bl_server_name = curr->ptr; + if(Curl_raw_nequal(bl_server_name, server_name, + strlen(bl_server_name))) { + infof(handle, "Server %s is blacklisted\n", server_name); + return TRUE; + } + curr = curr->next; + } + } + + infof(handle, "Server %s is not blacklisted\n", server_name); + } + return FALSE; +} + +CURLMcode Curl_pipeline_set_server_blacklist(char **servers, + struct curl_llist **list_ptr) +{ + struct curl_llist *old_list = *list_ptr; + struct curl_llist *new_list = NULL; + + if(servers) { + new_list = Curl_llist_alloc((curl_llist_dtor) server_blacklist_llist_dtor); + if(!new_list) + return CURLM_OUT_OF_MEMORY; + + /* Parse the URLs and populate the list */ + while(*servers) { + char *server_name; + + server_name = strdup(*servers); + if(!server_name) + return CURLM_OUT_OF_MEMORY; + + if(!Curl_llist_insert_next(new_list, new_list->tail, server_name)) + return CURLM_OUT_OF_MEMORY; + + servers++; + } + } + + /* Free the old list */ + if(old_list) { + Curl_llist_destroy(old_list, NULL); + } + + /* This might be NULL if sites == NULL, i.e the blacklist is cleared */ + *list_ptr = new_list; + + return CURLM_OK; +} + + +void print_pipeline(struct connectdata *conn) +{ + struct curl_llist_element *curr; + struct connectbundle *cb_ptr; + struct SessionHandle *data = conn->data; + + cb_ptr = conn->bundle; + + if(cb_ptr) { + curr = cb_ptr->conn_list->head; + while(curr) { + conn = curr->ptr; + infof(data, "- Conn %d (%p) send_pipe: %d, recv_pipe: %d\n", + conn->connection_id, + conn, + conn->send_pipe->size, + conn->recv_pipe->size); + curr = curr->next; + } + } +} diff --git a/lib/pipeline.h b/lib/pipeline.h new file mode 100644 index 000000000..f3a734c9a --- /dev/null +++ b/lib/pipeline.h @@ -0,0 +1,50 @@ +#ifndef HEADER_CURL_PIPELINE_H +#define HEADER_CURL_PIPELINE_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2013, Linus Nielsen Feltzing, <linus@haxx.se> + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +struct connectdata * +Curl_bundle_find_best(struct SessionHandle *data, + struct connectbundle *cb_ptr); + +CURLcode Curl_add_handle_to_pipeline(struct SessionHandle *handle, + struct connectdata *conn); +void Curl_move_handle_from_send_to_recv_pipe(struct SessionHandle *handle, + struct connectdata *conn); +bool Curl_pipeline_penalized(struct SessionHandle *data, + struct connectdata *conn); + +bool Curl_pipeline_site_blacklisted(struct SessionHandle *handle, + struct connectdata *conn); + +CURLMcode Curl_pipeline_set_site_blacklist(char **sites, + struct curl_llist **list_ptr); + +bool Curl_pipeline_server_blacklisted(struct SessionHandle *handle, + char *server_name); + +CURLMcode Curl_pipeline_set_server_blacklist(char **servers, + struct curl_llist **list_ptr); + +void print_pipeline(struct connectdata *conn); + +#endif /* HEADER_CURL_PIPELINE_H */ diff --git a/lib/sendf.c b/lib/sendf.c index c64d686b9..d5bf17282 100644 --- a/lib/sendf.c +++ b/lib/sendf.c @@ -529,8 +529,7 @@ CURLcode Curl_read(struct connectdata *conn, /* connection data */ ssize_t nread = 0; size_t bytesfromsocket = 0; char *buffertofill = NULL; - bool pipelining = (conn->data->multi && - Curl_multi_canPipeline(conn->data->multi)) ? TRUE : FALSE; + bool pipelining = Curl_multi_pipeline_enabled(conn->data->multi); /* Set 'num' to 0 or 1, depending on which socket that has been sent here. If it is the second socket, we set num to 1. Otherwise to 0. This lets diff --git a/lib/strerror.c b/lib/strerror.c index 58c6dc310..a385f5572 100644 --- a/lib/strerror.c +++ b/lib/strerror.c @@ -292,6 +292,9 @@ curl_easy_strerror(CURLcode error) case CURLE_CHUNK_FAILED: return "Chunk callback failed"; + case CURLE_NO_CONNECTION_AVAILABLE: + return "The max connection limit is reached"; + /* error codes not used by current libcurl */ case CURLE_OBSOLETE16: case CURLE_OBSOLETE20: diff --git a/lib/transfer.c b/lib/transfer.c index 330b37a2b..db0318d5a 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -473,7 +473,7 @@ static CURLcode readwrite_data(struct SessionHandle *data, /* We've stopped dealing with input, get out of the do-while loop */ if(nread > 0) { - if(conn->data->multi && Curl_multi_canPipeline(conn->data->multi)) { + if(Curl_multi_pipeline_enabled(conn->data->multi)) { infof(data, "Rewinding stream by : %zd" " bytes on url %s (zero-length body)\n", @@ -602,8 +602,7 @@ static CURLcode readwrite_data(struct SessionHandle *data, if(dataleft != 0) { infof(conn->data, "Leftovers after chunking: %zu bytes\n", dataleft); - if(conn->data->multi && - Curl_multi_canPipeline(conn->data->multi)) { + if(Curl_multi_pipeline_enabled(conn->data->multi)) { /* only attempt the rewind if we truly are pipelining */ infof(conn->data, "Rewinding %zu bytes\n",dataleft); read_rewind(conn, dataleft); @@ -626,7 +625,7 @@ static CURLcode readwrite_data(struct SessionHandle *data, excess = (size_t)(k->bytecount + nread - k->maxdownload); if(excess > 0 && !k->ignorebody) { - if(conn->data->multi && Curl_multi_canPipeline(conn->data->multi)) { + if(Curl_multi_pipeline_enabled(conn->data->multi)) { /* The 'excess' amount below can't be more than BUFSIZE which always will fit in a size_t */ infof(data, @@ -123,6 +123,7 @@ int curl_win32_idn_to_ascii(const char *in, char **out); #include "bundles.h" #include "conncache.h" #include "multihandle.h" +#include "pipeline.h" #define _MPRINTF_REPLACE /* use our functions only */ #include <curl/mprintf.h> @@ -134,6 +135,9 @@ int curl_win32_idn_to_ascii(const char *in, char **out); /* Local static prototypes */ static struct connectdata * find_oldest_idle_connection(struct SessionHandle *data); +static struct connectdata * +find_oldest_idle_connection_in_bundle(struct SessionHandle *data, + struct connectbundle *bundle); static void conn_free(struct connectdata *conn); static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke); static CURLcode do_init(struct connectdata *conn); @@ -2470,13 +2474,9 @@ static void conn_free(struct connectdata *conn) Curl_llist_destroy(conn->send_pipe, NULL); Curl_llist_destroy(conn->recv_pipe, NULL); - Curl_llist_destroy(conn->pend_pipe, NULL); - Curl_llist_destroy(conn->done_pipe, NULL); conn->send_pipe = NULL; conn->recv_pipe = NULL; - conn->pend_pipe = NULL; - conn->done_pipe = NULL; Curl_safefree(conn->localdev); Curl_free_ssl_config(&conn->ssl_config); @@ -2566,11 +2566,9 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) Curl_ssl_close(conn, FIRSTSOCKET); /* Indicate to all handles on the pipe that we're dead */ - if(Curl_isPipeliningEnabled(data)) { + if(Curl_multi_pipeline_enabled(data->multi)) { signalPipeClose(conn->send_pipe, TRUE); signalPipeClose(conn->recv_pipe, TRUE); - signalPipeClose(conn->pend_pipe, TRUE); - signalPipeClose(conn->done_pipe, FALSE); } conn_free(conn); @@ -2602,7 +2600,7 @@ static bool IsPipeliningPossible(const struct SessionHandle *handle, const struct connectdata *conn) { if((conn->handler->protocol & CURLPROTO_HTTP) && - handle->multi && Curl_multi_canPipeline(handle->multi) && + Curl_multi_pipeline_enabled(handle->multi) && (handle->set.httpreq == HTTPREQ_GET || handle->set.httpreq == HTTPREQ_HEAD) && handle->set.httpversion != CURL_HTTP_VERSION_1_0) @@ -2613,10 +2611,7 @@ static bool IsPipeliningPossible(const struct SessionHandle *handle, bool Curl_isPipeliningEnabled(const struct SessionHandle *handle) { - if(handle->multi && Curl_multi_canPipeline(handle->multi)) - return TRUE; - - return FALSE; + return Curl_multi_pipeline_enabled(handle->multi); } CURLcode Curl_addHandleToPipeline(struct SessionHandle *data, @@ -2624,6 +2619,7 @@ CURLcode Curl_addHandleToPipeline(struct SessionHandle *data, { if(!Curl_llist_insert_next(pipeline, pipeline->tail, data)) return CURLE_OUT_OF_MEMORY; + infof(data, "Curl_addHandleToPipeline: length: %d\n", pipeline->size); return CURLE_OK; } @@ -2683,8 +2679,6 @@ void Curl_getoff_all_pipelines(struct SessionHandle *data, conn->readchannel_inuse = FALSE; if(Curl_removeHandleFromPipeline(data, conn->send_pipe) && send_head) conn->writechannel_inuse = FALSE; - Curl_removeHandleFromPipeline(data, conn->pend_pipe); - Curl_removeHandleFromPipeline(data, conn->done_pipe); } static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) @@ -2715,8 +2709,8 @@ static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) } /* - * This function kills and removes an existing connection in the connection - * cache. The connection that has been unused for the longest time. + * This function finds the connection in the connection + * cache that has been unused for the longest time. * * Returns the pointer to the oldest idle connection, or NULL if none was * found. @@ -2767,6 +2761,47 @@ find_oldest_idle_connection(struct SessionHandle *data) } /* + * This function finds the connection in the connection + * bundle that has been unused for the longest time. + * + * Returns the pointer to the oldest idle connection, or NULL if none was + * found. + */ +static struct connectdata * +find_oldest_idle_connection_in_bundle(struct SessionHandle *data, + struct connectbundle *bundle) +{ + struct curl_llist_element *curr; + long highscore=-1; + long score; + struct timeval now; + struct connectdata *conn_candidate = NULL; + struct connectdata *conn; + + (void)data; + + now = Curl_tvnow(); + + curr = bundle->conn_list->head; + while(curr) { + conn = curr->ptr; + + if(!conn->inuse) { + /* Set higher score for the age passed since the connection was used */ + score = Curl_tvdiff(now, conn->now); + + if(score > highscore) { + highscore = score; + conn_candidate = conn; + } + } + curr = curr->next; + } + + return conn_candidate; +} + +/* * Given one filled in connection struct (named needle), this function should * detect if there already is one that has all the significant details * exactly the same and thus should be used instead. @@ -2774,11 +2809,15 @@ find_oldest_idle_connection(struct SessionHandle *data) * If there is a match, this function returns TRUE - and has marked the * connection as 'in-use'. It must later be called with ConnectionDone() to * return back to 'idle' (unused) state. + * + * The force_reuse flag is set if the connection must be used, even if + * the pipelining strategy wants to open a new connection instead of reusing. */ static bool ConnectionExists(struct SessionHandle *data, struct connectdata *needle, - struct connectdata **usethis) + struct connectdata **usethis, + bool *force_reuse) { struct connectdata *check; struct connectdata *chosen = 0; @@ -2787,15 +2826,30 @@ ConnectionExists(struct SessionHandle *data, (data->state.authhost.want==CURLAUTH_NTLM_WB) ? TRUE : FALSE; struct connectbundle *bundle; + *force_reuse = FALSE; + + /* We can't pipe if the site is blacklisted */ + if(canPipeline && Curl_pipeline_site_blacklisted(data, needle)) { + canPipeline = FALSE; + } + /* Look up the bundle with all the connections to this particular host */ bundle = Curl_conncache_find_bundle(data->state.conn_cache, needle->host.name); if(bundle) { + size_t max_pipe_len = Curl_multi_max_pipeline_length(data->multi); + size_t best_pipe_len = max_pipe_len; struct curl_llist_element *curr; infof(data, "Found bundle for host %s: %p\n", needle->host.name, bundle); + /* We can't pipe if we don't know anything about the server */ + if(canPipeline && !bundle->server_supports_pipelining) { + infof(data, "Server doesn't support pipelining\n"); + canPipeline = FALSE; + } + curr = bundle->conn_list->head; while(curr) { bool match = FALSE; @@ -2845,12 +2899,6 @@ ConnectionExists(struct SessionHandle *data, if(!IsPipeliningPossible(rh, check)) continue; } -#ifdef DEBUGBUILD - if(pipeLen > MAX_PIPELINE_LENGTH) { - infof(data, "BAD! Connection #%ld has too big pipeline!\n", - check->connection_id); - } -#endif } else { if(pipeLen > 0) { @@ -2989,26 +3037,60 @@ ConnectionExists(struct SessionHandle *data, } if(match) { - chosen = check; + /* If we are looking for an NTLM connection, check if this is already + authenticating with the right credentials. If not, keep looking so + that we can reuse NTLM connections if possible. (Especially we + must not reuse the same connection if partway through + a handshake!) */ + if(wantNTLM) { + if(credentialsMatch && check->ntlm.state != NTLMSTATE_NONE) { + chosen = check; + + /* We must use this connection, no other */ + *force_reuse = TRUE; + break; + } + else + continue; + } - /* If we are not looking for an NTLM connection, we can choose this one - immediately. */ - if(!wantNTLM) - break; + if(canPipeline) { + /* We can pipeline if we want to. Let's continue looking for + the optimal connection to use, i.e the shortest pipe that is not + blacklisted. */ - /* Otherwise, check if this is already authenticating with the right - credentials. If not, keep looking so that we can reuse NTLM - connections if possible. (Especially we must reuse the same - connection if partway through a handshake!) */ - if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE) + if(pipeLen == 0) { + /* We have the optimal connection. Let's stop looking. */ + chosen = check; + break; + } + + /* We can't use the connection if the pipe is full */ + if(pipeLen >= max_pipe_len) + continue; + + /* We can't use the connection if the pipe is penalized */ + if(Curl_pipeline_penalized(data, check)) + continue; + + if(pipeLen < best_pipe_len) { + /* This connection has a shorter pipe so far. We'll pick this + and continue searching */ + chosen = check; + best_pipe_len = pipeLen; + continue; + } + } + else { + /* We have found a connection. Let's stop searching. */ + chosen = check; break; + } } } } if(chosen) { - chosen->inuse = TRUE; /* mark this as being in use so that no other - handle in a multi stack may nick it */ *usethis = chosen; return TRUE; /* yes, we found one to use! */ } @@ -3475,7 +3557,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) conn->response_header = NULL; #endif - if(data->multi && Curl_multi_canPipeline(data->multi) && + if(Curl_multi_pipeline_enabled(data->multi) && !conn->master_buffer) { /* Allocate master_buffer to be used for pipelining */ conn->master_buffer = calloc(BUFSIZE, sizeof (char)); @@ -3486,10 +3568,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) /* Initialize the pipeline lists */ conn->send_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); conn->recv_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); - conn->pend_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); - conn->done_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor); - if(!conn->send_pipe || !conn->recv_pipe || !conn->pend_pipe || - !conn->done_pipe) + if(!conn->send_pipe || !conn->recv_pipe) goto error; #if defined(HAVE_KRB4) || defined(HAVE_GSSAPI) @@ -3515,13 +3594,9 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) Curl_llist_destroy(conn->send_pipe, NULL); Curl_llist_destroy(conn->recv_pipe, NULL); - Curl_llist_destroy(conn->pend_pipe, NULL); - Curl_llist_destroy(conn->done_pipe, NULL); conn->send_pipe = NULL; conn->recv_pipe = NULL; - conn->pend_pipe = NULL; - conn->done_pipe = NULL; Curl_safefree(conn->master_buffer); Curl_safefree(conn->localdev); @@ -4623,13 +4698,9 @@ static void reuse_conn(struct connectdata *old_conn, Curl_llist_destroy(old_conn->send_pipe, NULL); Curl_llist_destroy(old_conn->recv_pipe, NULL); - Curl_llist_destroy(old_conn->pend_pipe, NULL); - Curl_llist_destroy(old_conn->done_pipe, NULL); old_conn->send_pipe = NULL; old_conn->recv_pipe = NULL; - old_conn->pend_pipe = NULL; - old_conn->done_pipe = NULL; Curl_safefree(old_conn->master_buffer); } @@ -4663,6 +4734,10 @@ static CURLcode create_conn(struct SessionHandle *data, bool reuse; char *proxy = NULL; bool prot_missing = FALSE; + bool no_connections_available = FALSE; + bool force_reuse; + size_t max_host_connections = Curl_multi_max_host_connections(data->multi); + size_t max_total_connections = Curl_multi_max_total_connections(data->multi); *async = FALSE; @@ -4963,7 +5038,25 @@ static CURLcode create_conn(struct SessionHandle *data, if(data->set.reuse_fresh && !data->state.this_is_a_follow) reuse = FALSE; else - reuse = ConnectionExists(data, conn, &conn_temp); + reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse); + + /* If we found a reusable connection, we may still want to + open a new connection if we are pipelining. */ + if(reuse && !force_reuse && IsPipeliningPossible(data, conn_temp)) { + size_t pipelen = conn_temp->send_pipe->size + conn_temp->recv_pipe->size; + if(pipelen > 0) { + infof(data, "Found connection %d, with requests in the pipe (%d)\n", + conn_temp->connection_id, pipelen); + + if(conn_temp->bundle->num_connections < max_host_connections && + data->state.conn_cache->num_connections < max_total_connections) { + /* We want a new connection anyway */ + reuse = FALSE; + + infof(data, "We can reuse, but we want a new connection anyway\n"); + } + } + } if(reuse) { /* @@ -4972,6 +5065,8 @@ static CURLcode create_conn(struct SessionHandle *data, * just allocated before we can move along and use the previously * existing one. */ + conn_temp->inuse = TRUE; /* mark this as being in use so that no other + handle in a multi stack may nick it */ reuse_conn(conn, conn_temp); free(conn); /* we don't need this anymore */ conn = conn_temp; @@ -4985,14 +5080,66 @@ static CURLcode create_conn(struct SessionHandle *data, conn->proxy.name?conn->proxy.dispname:conn->host.dispname); } else { - /* - * This is a brand new connection, so let's store it in the connection - * cache of ours! - */ - conn->inuse = TRUE; - ConnectionStore(data, conn); + /* We have decided that we want a new connection. However, we may not + be able to do that if we have reached the limit of how many + connections we are allowed to open. */ + struct connectbundle *bundle; + + bundle = Curl_conncache_find_bundle(data->state.conn_cache, + conn->host.name); + if(max_host_connections > 0 && bundle && + (bundle->num_connections >= max_host_connections)) { + struct connectdata *conn_candidate; + + /* The bundle is full. Let's see if we can kill a connection. */ + conn_candidate = find_oldest_idle_connection_in_bundle(data, bundle); + + if(conn_candidate) { + /* Set the connection's owner correctly, then kill it */ + conn_candidate->data = data; + (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); + } + else + no_connections_available = TRUE; + } + + if(max_total_connections > 0 && + (data->state.conn_cache->num_connections >= max_total_connections)) { + struct connectdata *conn_candidate; + + /* The cache is full. Let's see if we can kill a connection. */ + conn_candidate = find_oldest_idle_connection(data); + + if(conn_candidate) { + /* Set the connection's owner correctly, then kill it */ + conn_candidate->data = data; + (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); + } + else + no_connections_available = TRUE; + } + + + if(no_connections_available) { + infof(data, "No connections available.\n"); + + conn_free(conn); + *in_connect = NULL; + + return CURLE_NO_CONNECTION_AVAILABLE; + } + else { + /* + * This is a brand new connection, so let's store it in the connection + * cache of ours! + */ + ConnectionStore(data, conn); + } } + /* Mark the connection as used */ + conn->inuse = TRUE; + /* Setup and init stuff before DO starts, in preparing for the transfer. */ do_init(conn); @@ -5167,6 +5314,11 @@ CURLcode Curl_connect(struct SessionHandle *data, } } + if(code == CURLE_NO_CONNECTION_AVAILABLE) { + *in_connect = NULL; + return code; + } + if(code && *in_connect) { /* We're not allowed to return failure with memory left allocated in the connectdata struct, free those here */ @@ -5258,12 +5410,8 @@ CURLcode Curl_done(struct connectdata **connp, state it is for re-using, so we're forced to close it. In a perfect world we can add code that keep track of if we really must close it here or not, but currently we have no such detail knowledge. - - connection_id == -1 here means that the connection has not been added - to the connection cache (OOM) and thus we must disconnect it here. */ - if(data->set.reuse_forbid || conn->bits.close || premature || - (-1 == conn->connection_id)) { + if(data->set.reuse_forbid || conn->bits.close || premature) { CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */ /* If we had an error already, make sure we return that one. But diff --git a/lib/urldata.h b/lib/urldata.h index 1cf7c38b0..b63d8eed6 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -935,17 +935,10 @@ struct connectdata { handle */ bool writechannel_inuse; /* whether the write channel is in use by an easy handle */ - bool server_supports_pipelining; /* TRUE if server supports pipelining, - set after first response */ struct curl_llist *send_pipe; /* List of handles waiting to send on this pipeline */ struct curl_llist *recv_pipe; /* List of handles waiting to read their responses on this pipeline */ - struct curl_llist *pend_pipe; /* List of pending handles on - this pipeline */ - struct curl_llist *done_pipe; /* Handles that are finished, but - still reference this connectdata */ -#define MAX_PIPELINE_LENGTH 5 char* master_buffer; /* The master buffer allocated on-demand; used for pipelining. */ size_t read_pos; /* Current read position in the master buffer */ @@ -1022,8 +1015,7 @@ struct connectdata { TUNNEL_CONNECT, /* CONNECT has been sent off */ TUNNEL_COMPLETE /* CONNECT response received completely */ } tunnel_state[2]; /* two separate ones to allow FTP */ - - struct connectbundle *bundle; /* The bundle we are member of */ + struct connectbundle *bundle; /* The bundle we are member of */ }; /* The end of connectdata. */ @@ -1172,13 +1164,6 @@ struct UrlState { /* buffers to store authentication data in, as parsed from input options */ struct timeval keeps_speed; /* for the progress meter really */ - struct connectdata *pending_conn; /* This points to the connection we want - to open when we are waiting in the - CONNECT_PEND state in the multi - interface. This to avoid recreating it - when we enter the CONNECT state again. - */ - struct connectdata *lastconnect; /* The last connection, NULL if undefined */ char *headerbuff; /* allocated buffer to store headers in */ diff --git a/tests/README b/tests/README index fff618ef2..3ddc1c93e 100644 --- a/tests/README +++ b/tests/README @@ -39,6 +39,7 @@ The cURL Test Suite 1.1 Requires to run perl (and a unix-style shell) + python (and a unix-style shell) diff (when a test fails, a diff is shown) stunnel (for HTTPS and FTPS tests) OpenSSH or SunSSH (for SCP, SFTP and SOCKS4/5 tests) diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am index f65fe0688..7c2e648f5 100644 --- a/tests/data/Makefile.am +++ b/tests/data/Makefile.am @@ -95,13 +95,14 @@ test1400 test1401 test1402 test1403 test1404 test1405 test1406 test1407 \ test1408 test1409 test1410 test1411 test1412 test1413 \ test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \ test1508 \ +test1900 test1901 test1902 test1903 \ test2000 test2001 test2002 test2003 test2004 test2005 test2006 test2007 \ test2008 test2009 test2010 test2011 test2012 test2013 test2014 test2015 \ test2016 test2017 test2018 test2019 test2020 test2021 test2022 \ test2023 test2024 test2025 \ test2026 test2027 test2028 \ test2029 test2030 test2031 \ -test2032 +test2032 test2033 EXTRA_DIST = $(TESTCASES) DISABLED diff --git a/tests/data/test1900 b/tests/data/test1900 new file mode 100644 index 000000000..857c6096b --- /dev/null +++ b/tests/data/test1900 @@ -0,0 +1,57 @@ +<testcase> +<info> +<keywords> +HTTP +pipelining +multi +</keywords> +</info> + +# Server-side +<reply> +<data> +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 4 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +Handle 1 Completed with status 0 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 +</data> +</reply> + +# Client-side +<client> +<server> +http-pipe +</server> +<tool> +lib1900 +</tool> + <name> +HTTP GET using pipelining + </name> + <command> +http://%HOSTIP:%HTTPPIPEPORT/ +</command> +<file name="log/urls.txt"> +0 1k.txt +1000 100k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +</client> + +# Verify data after the test has been "shot" +<verify> +</verify> +</testcase> diff --git a/tests/data/test1901 b/tests/data/test1901 new file mode 100644 index 000000000..bacf9cb09 --- /dev/null +++ b/tests/data/test1901 @@ -0,0 +1,58 @@ +<testcase> +<info> +<keywords> +HTTP +pipelining +multi +</keywords> +</info> + +# Server-side +<reply> +<data> +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 +Handle 4 Completed with status 0 +Handle 1 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +</data> +</reply> + +# Client-side +<client> +<server> +http-pipe +</server> +<tool> +lib1900 +</tool> + <name> +HTTP GET using pipelining, blacklisted site + </name> + <command> +http://%HOSTIP:%HTTPPIPEPORT/ +</command> +<file name="log/urls.txt"> +blacklist_site 127.0.0.1:%HTTPPIPEPORT +0 1k.txt +1000 100k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +</client> + +# Verify data after the test has been "shot" +<verify> +</verify> +</testcase> diff --git a/tests/data/test1902 b/tests/data/test1902 new file mode 100644 index 000000000..22f262176 --- /dev/null +++ b/tests/data/test1902 @@ -0,0 +1,57 @@ +<testcase> +<info> +<keywords> +HTTP +pipelining +multi +</keywords> +</info> + +# Server-side +<reply> +<data> +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 1 Completed with status 0 +Handle 4 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 +</data> +</reply> + +# Client-side +<client> +<server> +http-pipe +</server> +<tool> +lib1900 +</tool> + <name> +HTTP GET using pipelining, broken pipe + </name> + <command> +http://%HOSTIP:%HTTPPIPEPORT/ +</command> +<file name="log/urls.txt"> +0 1k.txt +1000 connection_close.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +0 1k.txt +</client> + +# Verify data after the test has been "shot" +<verify> +</verify> +</testcase> diff --git a/tests/data/test1903 b/tests/data/test1903 new file mode 100644 index 000000000..01efa67f8 --- /dev/null +++ b/tests/data/test1903 @@ -0,0 +1,57 @@ +<testcase> +<info> +<keywords> +HTTP +pipelining +multi +</keywords> +</info> + +# Server-side +<reply> +<data> +Adding handle 0 +Handle 0 Completed with status 0 +Adding handle 1 +Adding handle 2 +Adding handle 3 +Adding handle 4 +Adding handle 5 +Adding handle 6 +Handle 2 Completed with status 0 +Handle 3 Completed with status 0 +Handle 4 Completed with status 0 +Handle 5 Completed with status 0 +Handle 6 Completed with status 0 +Handle 1 Completed with status 0 +</data> +</reply> + +# Client-side +<client> +<server> +http-pipe +</server> +<tool> +lib1900 +</tool> + <name> +HTTP GET using pipelining, penalized on content-length + </name> + <command> +http://%HOSTIP:%HTTPPIPEPORT/ +</command> +<file name="log/urls.txt"> +0 1k.txt +1000 100k.txt +550 alphabet.txt +10 alphabet.txt +10 alphabet.txt +10 alphabet.txt +10 alphabet.txt +</client> + +# Verify data after the test has been "shot" +<verify> +</verify> +</testcase> diff --git a/tests/data/test2033 b/tests/data/test2033 new file mode 100644 index 000000000..ad926ebce --- /dev/null +++ b/tests/data/test2033 @@ -0,0 +1,144 @@ +<testcase> +<info> +<keywords> +HTTP +HTTP GET +HTTP Basic auth +HTTP NTLM auth +pipelining +</keywords> +</info> +# Server-side +<reply> + +<!-- Basic auth --> +<data100> +HTTP/1.1 401 Need Basic or NTLM auth +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 29 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is a bad password page! +</data100> + +<!-- NTML auth --> +<data200> +HTTP/1.1 401 Need Basic or NTLM auth (2) +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 27 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is not the real page! +</data200> + +<data1201> +HTTP/1.1 401 NTLM intermediate (2) +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 33 +WWW-Authenticate: NTLM TlRMTVNTUAACAAAACAAIADAAAAAGggEAq6U1NAWaJCIAAAAAAAAAAAAAAAA4AAAATlRMTUF1dGg= + +This is still not the real page! +</data1201> + +<data1202> +HTTP/1.1 200 Things are fine in server land +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 32 + +Finally, this is the real page! +</data1202> + +<datacheck> +HTTP/1.1 401 Need Basic or NTLM auth +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 29 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is a bad password page! +HTTP/1.1 401 Need Basic or NTLM auth +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 29 +WWW-Authenticate: NTLM +WWW-Authenticate: Basic realm="testrealm" + +This is a bad password page! +HTTP/1.1 401 NTLM intermediate (2) +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 33 +WWW-Authenticate: NTLM TlRMTVNTUAACAAAACAAIADAAAAAGggEAq6U1NAWaJCIAAAAAAAAAAAAAAAA4AAAATlRMTUF1dGg= + +HTTP/1.1 200 Things are fine in server land +Server: Microsoft-IIS/5.0 +Content-Type: text/html; charset=iso-8859-1 +Content-Length: 32 + +Finally, this is the real page! +</datacheck> + +</reply> + +# Client-side +<client> +<server> +http +</server> +<tool> +lib2033 +</tool> + + <name> +NTLM connection mapping, pipelining enabled + </name> + <setenv> +# we force our own host name, in order to make the test machine independent +CURL_GETHOSTNAME=curlhost +# we try to use the LD_PRELOAD hack, if not a debug build +LD_PRELOAD=%PWD/libtest/.libs/libhostname.so + </setenv> + <command> +http://%HOSTIP:%HTTPPORT/2032 +</command> +<precheck> +chkhostname curlhost +</precheck> +</client> + +# Verify data after the test has been "shot" +<verify> +<strip> +^User-Agent:.* +</strip> +<protocol> +GET /20320100 HTTP/1.1
+Authorization: Basic dGVzdHVzZXI6dGVzdHBhc3M=
+Host: 127.0.0.1:8990
+Accept: */*
+
+GET /20320100 HTTP/1.1
+Authorization: Basic dGVzdHVzZXI6dGVzdHBhc3M=
+Host: 127.0.0.1:8990
+Accept: */*
+
+GET /20320200 HTTP/1.1
+Authorization: NTLM TlRMTVNTUAABAAAABoIIAAAAAAAAAAAAAAAAAAAAAAA=
+Host: 127.0.0.1:8990
+Accept: */*
+
+GET /20320200 HTTP/1.1
+Authorization: NTLM TlRMTVNTUAADAAAAGAAYAEAAAAAYABgAWAAAAAAAAABwAAAACAAIAHAAAAAIAAgAeAAAAAAAAAAAAAAABoIBAI+/Fp9IERAQ74OsdNPbBpg7o8CVwLSO4DtFyIcZHUMKVktWIu92s2892OVpd2JzqnRlc3R1c2VyY3VybGhvc3Q=
+Host: 127.0.0.1:8990
+Accept: */*
+
+</protocol> +</verify> +</testcase> diff --git a/tests/data/test530 b/tests/data/test530 index 359d04cf1..09e742179 100644 --- a/tests/data/test530 +++ b/tests/data/test530 @@ -2,7 +2,7 @@ <info> <keywords> HTTP -Pipelining +pipelining multi </keywords> </info> diff --git a/tests/data/test536 b/tests/data/test536 index 334c07f06..ef8263d98 100644 --- a/tests/data/test536 +++ b/tests/data/test536 @@ -1,4 +1,12 @@ <testcase> +<info> +<keywords> +HTTP +pipelining +multi +</keywords> +</info> + <reply> <data mode="text"> HTTP/1.1 404 Badness diff --git a/tests/data/test584 b/tests/data/test584 index 81d6a083d..8d1ca92f9 100644 --- a/tests/data/test584 +++ b/tests/data/test584 @@ -2,7 +2,7 @@ <info> <keywords> HTTP -Pipelining +pipelining multi </keywords> </info> diff --git a/tests/http_pipe.py b/tests/http_pipe.py new file mode 100755 index 000000000..67185be8c --- /dev/null +++ b/tests/http_pipe.py @@ -0,0 +1,447 @@ +#!/usr/bin/python + +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Linus Nielsen Feltzing for inclusion in the libcurl test +# framework +# +import SocketServer +import argparse +import re +import select +import socket +import time +import pprint +import os + +INFO_MESSAGE = ''' +This is a test server to test the libcurl pipelining functionality. +It is a modified version if Google's HTTP pipelining test server. More +information can be found here: + +http://dev.chromium.org/developers/design-documents/network-stack/http-pipelining + +Source code can be found here: + +http://code.google.com/p/http-pipelining-test/ +''' +MAX_REQUEST_SIZE = 1024 # bytes +MIN_POLL_TIME = 0.01 # seconds. Minimum time to poll, in order to prevent + # excessive looping because Python refuses to poll for + # small timeouts. +SEND_BUFFER_TIME = 0.5 # seconds +TIMEOUT = 30 # seconds + + +class Error(Exception): + pass + + +class RequestTooLargeError(Error): + pass + + +class ServeIndexError(Error): + pass + + +class UnexpectedMethodError(Error): + pass + + +class RequestParser(object): + """Parses an input buffer looking for HTTP GET requests.""" + + global logfile + + LOOKING_FOR_GET = 1 + READING_HEADERS = 2 + + HEADER_RE = re.compile('([^:]+):(.*)\n') + REQUEST_RE = re.compile('([^ ]+) ([^ ]+) HTTP/(\d+)\.(\d+)\n') + + def __init__(self): + """Initializer.""" + self._buffer = "" + self._pending_headers = {} + self._pending_request = "" + self._state = self.LOOKING_FOR_GET + self._were_all_requests_http_1_1 = True + self._valid_requests = [] + + def ParseAdditionalData(self, data): + """Finds HTTP requests in |data|. + + Args: + data: (String) Newly received input data from the socket. + + Returns: + (List of Tuples) + (String) The request path. + (Map of String to String) The header name and value. + + Raises: + RequestTooLargeError: If the request exceeds MAX_REQUEST_SIZE. + UnexpectedMethodError: On a non-GET method. + Error: On a programming error. + """ + logfile = open('log/server.input', 'a') + logfile.write(data) + logfile.close() + self._buffer += data.replace('\r', '') + should_continue_parsing = True + while should_continue_parsing: + if self._state == self.LOOKING_FOR_GET: + should_continue_parsing = self._DoLookForGet() + elif self._state == self.READING_HEADERS: + should_continue_parsing = self._DoReadHeader() + else: + raise Error('Unexpected state: ' + self._state) + if len(self._buffer) > MAX_REQUEST_SIZE: + raise RequestTooLargeError( + 'Request is at least %d bytes' % len(self._buffer)) + valid_requests = self._valid_requests + self._valid_requests = [] + return valid_requests + + @property + def were_all_requests_http_1_1(self): + return self._were_all_requests_http_1_1 + + def _DoLookForGet(self): + """Tries to parse an HTTTP request line. + + Returns: + (Boolean) True if a request was found. + + Raises: + UnexpectedMethodError: On a non-GET method. + """ + m = self.REQUEST_RE.match(self._buffer) + if not m: + return False + method, path, http_major, http_minor = m.groups() + + if method != 'GET': + raise UnexpectedMethodError('Unexpected method: ' + method) + if path in ['/', '/index.htm', '/index.html']: + raise ServeIndexError() + + if http_major != '1' or http_minor != '1': + self._were_all_requests_http_1_1 = False + +# print method, path + + self._pending_request = path + self._buffer = self._buffer[m.end():] + self._state = self.READING_HEADERS + return True + + def _DoReadHeader(self): + """Tries to parse a HTTP header. + + Returns: + (Boolean) True if it found the end of the request or a HTTP header. + """ + if self._buffer.startswith('\n'): + self._buffer = self._buffer[1:] + self._state = self.LOOKING_FOR_GET + self._valid_requests.append((self._pending_request, + self._pending_headers)) + self._pending_headers = {} + self._pending_request = "" + return True + + m = self.HEADER_RE.match(self._buffer) + if not m: + return False + + header = m.group(1).lower() + value = m.group(2).strip().lower() + if header not in self._pending_headers: + self._pending_headers[header] = value + self._buffer = self._buffer[m.end():] + return True + + +class ResponseBuilder(object): + """Builds HTTP responses for a list of accumulated requests.""" + + def __init__(self): + """Initializer.""" + self._max_pipeline_depth = 0 + self._requested_paths = [] + self._processed_end = False + self._were_all_requests_http_1_1 = True + + def QueueRequests(self, requested_paths, were_all_requests_http_1_1): + """Adds requests to the queue of requests. + + Args: + requested_paths: (List of Strings) Requested paths. + """ + self._requested_paths.extend(requested_paths) + self._were_all_requests_http_1_1 = were_all_requests_http_1_1 + + def Chunkify(self, data, chunksize): + """ Divides a string into chunks + """ + return [hex(chunksize)[2:] + "\r\n" + data[i:i+chunksize] + "\r\n" for i in range(0, len(data), chunksize)] + + def BuildResponses(self): + """Converts the queue of requests into responses. + + Returns: + (String) Buffer containing all of the responses. + """ + result = "" + self._max_pipeline_depth = max(self._max_pipeline_depth, + len(self._requested_paths)) + for path, headers in self._requested_paths: + if path == '/verifiedserver': + body = "WE ROOLZ: {}\r\n".format(os.getpid()); + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: {}'.format(len(body)), + 'Cache-Control: no-store'], body) + + elif path == '/alphabet.txt': + body = 'abcdefghijklmnopqrstuvwxyz' + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: 26', + 'Cache-Control: no-store'], body) + + elif path == '/reverse.txt': + body = 'zyxwvutsrqponmlkjihgfedcba' + result += self._BuildResponse( + '200 OK', ['Content-Length: 26', 'Cache-Control: no-store'], body) + + elif path == '/chunked.txt': + body = ('7\r\nchunked\r\n' + '8\r\nencoding\r\n' + '2\r\nis\r\n' + '3\r\nfun\r\n' + '0\r\n\r\n') + result += self._BuildResponse( + '200 OK', ['Transfer-Encoding: chunked', 'Cache-Control: no-store'], + body) + + elif path == '/cached.txt': + body = 'azbycxdwevfugthsirjqkplomn' + result += self._BuildResponse( + '200 OK', ['Content-Length: 26', 'Cache-Control: max-age=60'], body) + + elif path == '/connection_close.txt': + body = 'azbycxdwevfugthsirjqkplomn' + result += self._BuildResponse( + '200 OK', ['Content-Length: 26', 'Cache-Control: max-age=60', 'Connection: close'], body) + self._processed_end = True + + elif path == '/1k.txt': + str = '0123456789abcdef' + body = ''.join([str for num in xrange(64)]) + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: 1024', + 'Cache-Control: max-age=60'], body) + + elif path == '/10k.txt': + str = '0123456789abcdef' + body = ''.join([str for num in xrange(640)]) + result += self._BuildResponse( + '200 OK', ['Server: Apache', + 'Content-Length: 10240', + 'Cache-Control: max-age=60'], body) + + elif path == '/100k.txt': + str = '0123456789abcdef' + body = ''.join([str for num in xrange(6400)]) + result += self._BuildResponse( + '200 OK', + ['Server: Apache', + 'Content-Length: 102400', + 'Cache-Control: max-age=60'], + body) + + elif path == '/100k_chunked.txt': + str = '0123456789abcdef' + moo = ''.join([str for num in xrange(6400)]) + body = self.Chunkify(moo, 20480) + body.append('0\r\n\r\n') + body = ''.join(body) + + result += self._BuildResponse( + '200 OK', ['Transfer-Encoding: chunked', 'Cache-Control: no-store'], body) + + elif path == '/stats.txt': + results = { + 'max_pipeline_depth': self._max_pipeline_depth, + 'were_all_requests_http_1_1': int(self._were_all_requests_http_1_1), + } + body = ','.join(['%s:%s' % (k, v) for k, v in results.items()]) + result += self._BuildResponse( + '200 OK', + ['Content-Length: %s' % len(body), 'Cache-Control: no-store'], body) + self._processed_end = True + + else: + result += self._BuildResponse('404 Not Found', ['Content-Length: 7'], 'Go away') + if self._processed_end: + break + self._requested_paths = [] + return result + + def WriteError(self, status, error): + """Returns an HTTP response for the specified error. + + Args: + status: (String) Response code and descrtion (e.g. "404 Not Found") + + Returns: + (String) Text of HTTP response. + """ + return self._BuildResponse( + status, ['Connection: close', 'Content-Type: text/plain'], error) + + @property + def processed_end(self): + return self._processed_end + + def _BuildResponse(self, status, headers, body): + """Builds an HTTP response. + + Args: + status: (String) Response code and descrtion (e.g. "200 OK") + headers: (List of Strings) Headers (e.g. "Connection: close") + body: (String) Response body. + + Returns: + (String) Text of HTTP response. + """ + return ('HTTP/1.1 %s\r\n' + '%s\r\n' + '\r\n' + '%s' % (status, '\r\n'.join(headers), body)) + + +class PipelineRequestHandler(SocketServer.BaseRequestHandler): + """Called on an incoming TCP connection.""" + + def _GetTimeUntilTimeout(self): + return self._start_time + TIMEOUT - time.time() + + def _GetTimeUntilNextSend(self): + if not self._last_queued_time: + return TIMEOUT + return self._last_queued_time + SEND_BUFFER_TIME - time.time() + + def handle(self): + self._request_parser = RequestParser() + self._response_builder = ResponseBuilder() + self._last_queued_time = 0 + self._num_queued = 0 + self._num_written = 0 + self._send_buffer = "" + self._start_time = time.time() + try: + poller = select.epoll(sizehint=1) + poller.register(self.request.fileno(), select.EPOLLIN) + while not self._response_builder.processed_end or self._send_buffer: + + time_left = self._GetTimeUntilTimeout() + time_until_next_send = self._GetTimeUntilNextSend() + max_poll_time = min(time_left, time_until_next_send) + MIN_POLL_TIME + + events = None + if max_poll_time > 0: + if self._send_buffer: + poller.modify(self.request.fileno(), + select.EPOLLIN | select.EPOLLOUT) + else: + poller.modify(self.request.fileno(), select.EPOLLIN) + events = poller.poll(timeout=max_poll_time) + + if self._GetTimeUntilTimeout() <= 0: + return + + if self._GetTimeUntilNextSend() <= 0: + self._send_buffer += self._response_builder.BuildResponses() + self._num_written = self._num_queued + self._last_queued_time = 0 + + for fd, mode in events: + if mode & select.EPOLLIN: + new_data = self.request.recv(MAX_REQUEST_SIZE, socket.MSG_DONTWAIT) + if not new_data: + return + new_requests = self._request_parser.ParseAdditionalData(new_data) + self._response_builder.QueueRequests( + new_requests, self._request_parser.were_all_requests_http_1_1) + self._num_queued += len(new_requests) + self._last_queued_time = time.time() + elif mode & select.EPOLLOUT: + num_bytes_sent = self.request.send(self._send_buffer[0:4096]) + self._send_buffer = self._send_buffer[num_bytes_sent:] + time.sleep(0.05) + else: + return + + except RequestTooLargeError as e: + self.request.send(self._response_builder.WriteError( + '413 Request Entity Too Large', e)) + raise + except UnexpectedMethodError as e: + self.request.send(self._response_builder.WriteError( + '405 Method Not Allowed', e)) + raise + except ServeIndexError: + self.request.send(self._response_builder.WriteError( + '200 OK', INFO_MESSAGE)) + except Exception as e: + print e + self.request.close() + + +class PipelineServer(SocketServer.ForkingMixIn, SocketServer.TCPServer): + pass + + +parser = argparse.ArgumentParser() +parser.add_argument("--port", action="store", default=0, + type=int, help="port to listen on") +parser.add_argument("--verbose", action="store", default=0, + type=int, help="verbose output") +parser.add_argument("--pidfile", action="store", default=0, + help="file name for the PID") +parser.add_argument("--logfile", action="store", default=0, + help="file name for the log") +parser.add_argument("--srcdir", action="store", default=0, + help="test directory") +parser.add_argument("--id", action="store", default=0, + help="server ID") +parser.add_argument("--ipv4", action="store_true", default=0, + help="IPv4 flag") +args = parser.parse_args() + +if args.pidfile: + pid = os.getpid() + f = open(args.pidfile, 'w') + f.write('{}'.format(pid)) + f.close() + +server = PipelineServer(('0.0.0.0', args.port), PipelineRequestHandler) +server.allow_reuse_address = True +server.serve_forever() diff --git a/tests/libtest/.gitignore b/tests/libtest/.gitignore index 28e83c08d..7fd6e7e30 100644 --- a/tests/libtest/.gitignore +++ b/tests/libtest/.gitignore @@ -1,5 +1,7 @@ chkhostname lib5[0-9][0-9] lib150[0-9] +lib19[0-9][0-9] +lib2033 libauthretry libntlmconnect diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc index 88fc7d8fa..391c6255e 100644 --- a/tests/libtest/Makefile.inc +++ b/tests/libtest/Makefile.inc @@ -23,7 +23,9 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect \ lib582 lib583 lib585 lib586 lib587 \ lib590 lib591 lib597 lib598 lib599 \ \ - lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 + lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 \ + lib1900 \ + lib2033 chkhostname_SOURCES = chkhostname.c ../../lib/curl_gethostname.c chkhostname_LDADD = @CURL_NETWORK_LIBS@ @@ -323,3 +325,9 @@ lib1507_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1507 lib1508_SOURCES = lib1508.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS) lib1508_LDADD = $(TESTUTIL_LIBS) lib1508_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1508 + +lib1900_SOURCES = lib1900.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS) +lib1900_CPPFLAGS = $(AM_CPPFLAGS) + +lib2033_SOURCES = libntlmconnect.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS) +lib2033_CPPFLAGS = $(AM_CPPFLAGS) -DUSE_PIPELINING diff --git a/tests/libtest/lib1900.c b/tests/libtest/lib1900.c new file mode 100644 index 000000000..b2a943440 --- /dev/null +++ b/tests/libtest/lib1900.c @@ -0,0 +1,256 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2013, Linus Nielsen Feltzing, <linus@haxx.se> + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#include "test.h" + +#include "testutil.h" +#include "warnless.h" +#include "memdebug.h" + +#define TEST_HANG_TIMEOUT 60 * 1000 +#define MAX_URLS 200 +#define MAX_BLACKLIST 20 + +int urltime[MAX_URLS]; +char *urlstring[MAX_URLS]; +CURL *handles[MAX_URLS]; +char *site_blacklist[MAX_BLACKLIST]; +char *server_blacklist[MAX_BLACKLIST]; +int num_handles; +int blacklist_num_servers; +int blacklist_num_sites; + +int parse_url_file(const char *filename); +void free_urls(void); +int create_handles(void); +void setup_handle(char *base_url, CURLM *m, int handlenum); +void remove_handles(void); + +static size_t +write_callback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + (void)contents; + (void)userp; + + return realsize; +} + +int parse_url_file(const char *filename) +{ + FILE *f; + int time; + char buf[200]; + + num_handles = 0; + blacklist_num_sites = 0; + blacklist_num_servers = 0; + + f = fopen(filename, "rb"); + if(!f) + return 0; + + while(!feof(f)) { + if(fscanf(f, "%d %s\n", &time, buf)) { + urltime[num_handles] = time; + urlstring[num_handles] = strdup(buf); + num_handles++; + continue; + } + + if(fscanf(f, "blacklist_site %s\n", buf)) { + site_blacklist[blacklist_num_sites] = strdup(buf); + blacklist_num_sites++; + continue; + } + + break; + } + fclose(f); + + site_blacklist[blacklist_num_sites] = NULL; + server_blacklist[blacklist_num_servers] = NULL; + return num_handles; +} + +void free_urls(void) +{ + int i; + for(i = 0;i < num_handles;i++) { + free(urlstring[i]); + } + for(i = 0;i < blacklist_num_servers;i++) { + free(server_blacklist[i]); + } + for(i = 0;i < blacklist_num_sites;i++) { + free(site_blacklist[i]); + } +} + +int create_handles(void) +{ + int i; + + for(i = 0;i < num_handles;i++) { + handles[i] = curl_easy_init(); + } + return 0; +} + +void setup_handle(char *base_url, CURLM *m, int handlenum) +{ + char urlbuf[256]; + + sprintf(urlbuf, "%s%s", base_url, urlstring[handlenum]); + curl_easy_setopt(handles[handlenum], CURLOPT_URL, urlbuf); + curl_easy_setopt(handles[handlenum], CURLOPT_VERBOSE, 1L); + curl_easy_setopt(handles[handlenum], CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(handles[handlenum], CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(handles[handlenum], CURLOPT_WRITEDATA, NULL); + curl_multi_add_handle(m, handles[handlenum]); +} + +void remove_handles(void) +{ + int i; + + for(i = 0;i < num_handles;i++) { + if(handles[i]) + curl_easy_cleanup(handles[i]); + } +} + +int test(char *URL) +{ + int res = 0; + CURLM *m = NULL; + CURLMsg *msg; /* for picking up messages with the transfer status */ + int msgs_left; /* how many messages are left */ + int running; + int handlenum = 0; + struct timeval last_handle_add; + + if(parse_url_file("log/urls.txt") <= 0) + goto test_cleanup; + + start_test_timing(); + + curl_global_init(CURL_GLOBAL_ALL); + + m = curl_multi_init(); + + create_handles(); + + multi_setopt(m, CURLMOPT_PIPELINING, 1L); + multi_setopt(m, CURLMOPT_MAX_HOST_CONNECTIONS, 2L); + multi_setopt(m, CURLMOPT_MAX_PIPELINE_LENGTH, 3L); + multi_setopt(m, CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE, 15000L); + multi_setopt(m, CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE, 10000L); + + multi_setopt(m, CURLMOPT_PIPELINING_SITE_BL, site_blacklist); + multi_setopt(m, CURLMOPT_PIPELINING_SERVER_BL, server_blacklist); + + gettimeofday(&last_handle_add, NULL); + + for(;;) { + struct timeval interval; + struct timeval now; + long int msnow, mslast; + fd_set rd, wr, exc; + int maxfd = -99; + long timeout; + + interval.tv_sec = 1; + interval.tv_usec = 0; + + if(handlenum < num_handles) { + gettimeofday(&now, NULL); + msnow = now.tv_sec * 1000 + now.tv_usec / 1000; + mslast = last_handle_add.tv_sec * 1000 + last_handle_add.tv_usec / 1000; + if(msnow - mslast >= urltime[handlenum] && handlenum < num_handles) { + fprintf(stdout, "Adding handle %d\n", handlenum); + setup_handle(URL, m, handlenum); + last_handle_add = now; + handlenum++; + } + } + + curl_multi_perform(m, &running); + + abort_on_test_timeout(); + + /* See how the transfers went */ + while ((msg = curl_multi_info_read(m, &msgs_left))) { + if (msg->msg == CURLMSG_DONE) { + int i, found = 0; + + /* Find out which handle this message is about */ + for (i = 0; i < num_handles; i++) { + found = (msg->easy_handle == handles[i]); + if(found) + break; + } + + printf("Handle %d Completed with status %d\n", i, msg->data.result); + curl_multi_remove_handle(m, handles[i]); + } + } + + if(handlenum == num_handles && !running) { + break; /* done */ + } + + FD_ZERO(&rd); + FD_ZERO(&wr); + FD_ZERO(&exc); + + curl_multi_fdset(m, &rd, &wr, &exc, &maxfd); + + /* At this point, maxfd is guaranteed to be greater or equal than -1. */ + + curl_multi_timeout(m, &timeout); + + if(timeout < 0) + timeout = 1; + + interval.tv_sec = timeout / 1000; + interval.tv_usec = (timeout % 1000) * 1000; + + interval.tv_sec = 0; + interval.tv_usec = 1000; + + select_test(maxfd+1, &rd, &wr, &exc, &interval); + + abort_on_test_timeout(); + } + +test_cleanup: + + remove_handles(); + + /* undocumented cleanup sequence - type UB */ + + curl_multi_cleanup(m); + curl_global_cleanup(); + + free_urls(); + return res; +} diff --git a/tests/libtest/lib530.c b/tests/libtest/lib530.c index ad84ff8a5..06a846439 100644 --- a/tests/libtest/lib530.c +++ b/tests/libtest/lib530.c @@ -37,6 +37,7 @@ int test(char *URL) CURLM *m = NULL; int i; char target_url[256]; + int handles_added = 0; for(i=0; i < NUM_HANDLES; i++) curl[i] = NULL; @@ -59,10 +60,13 @@ int test(char *URL) easy_setopt(curl[i], CURLOPT_VERBOSE, 1L); /* include headers */ easy_setopt(curl[i], CURLOPT_HEADER, 1L); - /* add handle to multi */ - multi_add_handle(m, curl[i]); } + /* Add the first handle to multi. We do this to let libcurl detect + that the server can do pipelining. The rest of the handles will be + added later. */ + multi_add_handle(m, curl[handles_added++]); + multi_setopt(m, CURLMOPT_PIPELINING, 1L); fprintf(stderr, "Start at URL 0\n"); @@ -79,9 +83,14 @@ int test(char *URL) abort_on_test_timeout(); - if(!running) + if(!running && handles_added >= NUM_HANDLES) break; /* done */ + /* Add the rest of the handles now that the first handle has sent the + request. */ + while(handles_added < NUM_HANDLES) + multi_add_handle(m, curl[handles_added++]); + FD_ZERO(&rd); FD_ZERO(&wr); FD_ZERO(&exc); diff --git a/tests/libtest/libntlmconnect.c b/tests/libtest/libntlmconnect.c index b540ebf58..cd507dfa1 100644 --- a/tests/libtest/libntlmconnect.c +++ b/tests/libtest/libntlmconnect.c @@ -125,6 +125,12 @@ int test(char *url) multi_init(multi); +#ifdef USE_PIPELINING + multi_setopt(multi, CURLMOPT_PIPELINING, 1); + multi_setopt(multi, CURLMOPT_MAX_HOST_CONNECTIONS, 5); + multi_setopt(multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 10); +#endif + for(;;) { struct timeval interval; fd_set fdread; diff --git a/tests/runtests.pl b/tests/runtests.pl index 4915f2e81..cc6999cdc 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -140,6 +140,7 @@ my $GOPHER6PORT; # Gopher IPv6 server port my $HTTPTLSPORT; # HTTP TLS (non-stunnel) server port my $HTTPTLS6PORT; # HTTP TLS (non-stunnel) IPv6 server port my $HTTPPROXYPORT; # HTTP proxy port, when using CONNECT +my $HTTPPIPEPORT; # HTTP pipelining port my $srcdir = $ENV{'srcdir'} || '.'; my $CURL="../src/curl".exe_ext(); # what curl executable to run on the tests @@ -339,10 +340,10 @@ delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'}); # Load serverpidfile hash with pidfile names for all possible servers. # sub init_serverpidfile_hash { - for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp')) { + for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp', 'http')) { for my $ssl (('', 's')) { for my $ipvnum ((4, 6)) { - for my $idnum ((1, 2)) { + for my $idnum ((1, 2, 3)) { my $serv = servername_id("$proto$ssl", $ipvnum, $idnum); my $pidf = server_pidfilename("$proto$ssl", $ipvnum, $idnum); $serverpidfile{$serv} = $pidf; @@ -642,11 +643,11 @@ sub stopserver { # All servers relative to the given one must be stopped also # my @killservers; - if($server =~ /^(ftp|http|imap|pop3|smtp)s((\d*)(-ipv6|))$/) { + if($server =~ /^(ftp|http|imap|pop3|smtp|httppipe)s((\d*)(-ipv6|))$/) { # given a stunnel based ssl server, also kill non-ssl underlying one push @killservers, "${1}${2}"; } - elsif($server =~ /^(ftp|http|imap|pop3|smtp)((\d*)(-ipv6|))$/) { + elsif($server =~ /^(ftp|http|imap|pop3|smtp|httppipe)((\d*)(-ipv6|))$/) { # given a non-ssl server, also kill stunnel based ssl piggybacking one push @killservers, "${1}s${2}"; } @@ -1105,6 +1106,7 @@ my %protofunc = ('http' => \&verifyhttp, 'pop3' => \&verifyftp, 'imap' => \&verifyftp, 'smtp' => \&verifyftp, + 'httppipe' => \&verifyhttp, 'ftps' => \&verifyftp, 'tftp' => \&verifyftp, 'ssh' => \&verifyssh, @@ -1170,6 +1172,7 @@ sub runhttpserver { my $pidfile; my $logfile; my $flags = ""; + my $exe = "$perl $srcdir/httpserver.pl"; if($alt eq "ipv6") { # if IPv6, use a different setup @@ -1180,6 +1183,11 @@ sub runhttpserver { # basically the same, but another ID $idnum = 2; } + elsif($alt eq "pipe") { + # basically the same, but another ID + $idnum = 3; + $exe = "python $srcdir/http_pipe.py"; + } $server = servername_id($proto, $ipvnum, $idnum); @@ -1207,7 +1215,82 @@ sub runhttpserver { $flags .= "--id $idnum " if($idnum > 1); $flags .= "--ipv$ipvnum --port $port --srcdir \"$srcdir\""; - my $cmd = "$perl $srcdir/httpserver.pl $flags"; + my $cmd = "$exe $flags"; + my ($httppid, $pid2) = startnew($cmd, $pidfile, 15, 0); + + if($httppid <= 0 || !kill(0, $httppid)) { + # it is NOT alive + logmsg "RUN: failed to start the $srvrname server\n"; + stopserver($server, "$pid2"); + displaylogs($testnumcheck); + $doesntrun{$pidfile} = 1; + return (0,0); + } + + # Server is up. Verify that we can speak to it. + my $pid3 = verifyserver($proto, $ipvnum, $idnum, $ip, $port); + if(!$pid3) { + logmsg "RUN: $srvrname server failed verification\n"; + # failed to talk to it properly. Kill the server and return failure + stopserver($server, "$httppid $pid2"); + displaylogs($testnumcheck); + $doesntrun{$pidfile} = 1; + return (0,0); + } + $pid2 = $pid3; + + if($verbose) { + logmsg "RUN: $srvrname server is now running PID $httppid\n"; + } + + sleep(1); + + return ($httppid, $pid2); +} + +####################################################################### +# start the http server +# +sub runhttp_pipeserver { + my ($proto, $verbose, $alt, $port) = @_; + my $ip = $HOSTIP; + my $ipvnum = 4; + my $idnum = 1; + my $server; + my $srvrname; + my $pidfile; + my $logfile; + my $flags = ""; + + if($alt eq "ipv6") { + # No IPv6 + } + + $server = servername_id($proto, $ipvnum, $idnum); + + $pidfile = $serverpidfile{$server}; + + # don't retry if the server doesn't work + if ($doesntrun{$pidfile}) { + return (0,0); + } + + my $pid = processexists($pidfile); + if($pid > 0) { + stopserver($server, "$pid"); + } + unlink($pidfile) if(-f $pidfile); + + $srvrname = servername_str($proto, $ipvnum, $idnum); + + $logfile = server_logfilename($LOGDIR, $proto, $ipvnum, $idnum); + + $flags .= "--verbose " if($debugprotocol); + $flags .= "--pidfile \"$pidfile\" --logfile \"$logfile\" "; + $flags .= "--id $idnum " if($idnum > 1); + $flags .= "--port $port --srcdir \"$srcdir\""; + + my $cmd = "$srcdir/http_pipe.py $flags"; my ($httppid, $pid2) = startnew($cmd, $pidfile, 15, 0); if($httppid <= 0 || !kill(0, $httppid)) { @@ -2276,6 +2359,9 @@ sub checksystem { # 'http-proxy' is used in test cases to do CONNECT through push @protocols, 'http-proxy'; + # 'http-pipe' is the special server for testing pipelining + push @protocols, 'http-pipe'; + # 'none' is used in test cases to mean no server push @protocols, 'none'; } @@ -2477,6 +2563,7 @@ sub checksystem { } logmsg "\n"; } + logmsg sprintf("* HTTP-PIPE/%d \n", $HTTPPIPEPORT); $has_textaware = ($^O eq 'MSWin32') || ($^O eq 'msys'); @@ -2505,6 +2592,7 @@ sub subVariables { $$thing =~ s/%HTTP6PORT/$HTTP6PORT/g; $$thing =~ s/%HTTPSPORT/$HTTPSPORT/g; $$thing =~ s/%HTTPPORT/$HTTPPORT/g; + $$thing =~ s/%HTTPPIPEPORT/$HTTPPIPEPORT/g; $$thing =~ s/%PROXYPORT/$HTTPPROXYPORT/g; $$thing =~ s/%IMAP6PORT/$IMAP6PORT/g; @@ -3870,6 +3958,23 @@ sub startservers { $run{'http-ipv6'}="$pid $pid2"; } } + elsif($what eq "http-pipe") { + if($torture && $run{'http-pipe'} && + !responsive_http_server("http", $verbose, "pipe", + $HTTPPIPEPORT)) { + stopserver('http-pipe'); + } + if(!$run{'http-pipe'}) { + ($pid, $pid2) = runhttpserver("http", $verbose, "pipe", + $HTTPPIPEPORT); + if($pid <= 0) { + return "failed starting HTTP-pipe server"; + } + logmsg sprintf ("* pid http-pipe => %d %d\n", $pid, $pid2) + if($verbose); + $run{'http-pipe'}="$pid $pid2"; + } + } elsif($what eq "rtsp") { if($torture && $run{'rtsp'} && !responsive_rtsp_server($verbose)) { @@ -4512,6 +4617,7 @@ $GOPHER6PORT = $base++; # Gopher IPv6 server port $HTTPTLSPORT = $base++; # HTTP TLS (non-stunnel) server port $HTTPTLS6PORT = $base++; # HTTP TLS (non-stunnel) IPv6 server port $HTTPPROXYPORT = $base++; # HTTP proxy port, when using CONNECT +$HTTPPIPEPORT = $base++; # HTTP pipelining port ####################################################################### # clear and create logging directory: diff --git a/tests/serverhelp.pm b/tests/serverhelp.pm index a1d1dc367..b0b5b7492 100644 --- a/tests/serverhelp.pm +++ b/tests/serverhelp.pm @@ -79,7 +79,7 @@ sub serverfactors { my $idnum; if($server =~ - /^((ftp|http|imap|pop3|smtp)s?)(\d*)(-ipv6|)$/) { + /^((ftp|http|imap|pop3|smtp|http-pipe)s?)(\d*)(-ipv6|)$/) { $proto = $1; $idnum = ($3 && ($3 > 1)) ? $3 : 1; $ipvnum = ($4 && ($4 =~ /6$/)) ? 6 : 4; @@ -105,7 +105,7 @@ sub servername_str { $proto = uc($proto) if($proto); die "unsupported protocol: '$proto'" unless($proto && - ($proto =~ /^(((FTP|HTTP|IMAP|POP3|SMTP)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS))$/)); + ($proto =~ /^(((FTP|HTTP|IMAP|POP3|SMTP|HTTP-PIPE)S?)|(TFTP|SFTP|SOCKS|SSH|RTSP|GOPHER|HTTPTLS))$/)); $ipver = (not $ipver) ? 'ipv4' : lc($ipver); die "unsupported IP version: '$ipver'" unless($ipver && |