diff options
| -rw-r--r-- | debug/shared-conn.c | 68 | ||||
| -rw-r--r-- | lib/conncache.c | 141 | ||||
| -rw-r--r-- | lib/conncache.h | 10 | ||||
| -rwxr-xr-x | lib/connect.c | 2 | ||||
| -rw-r--r-- | lib/multi.c | 62 | ||||
| -rw-r--r-- | lib/multihandle.h | 4 | ||||
| -rw-r--r-- | lib/share.c | 6 | ||||
| -rw-r--r-- | lib/share.h | 5 | ||||
| -rw-r--r-- | lib/url.c | 56 | ||||
| -rw-r--r-- | lib/url.h | 4 |
10 files changed, 230 insertions, 128 deletions
diff --git a/debug/shared-conn.c b/debug/shared-conn.c new file mode 100644 index 000000000..f259a8c01 --- /dev/null +++ b/debug/shared-conn.c @@ -0,0 +1,68 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +/* <DESC> + * Two HTTP GET using connection sharing with the share inteface + * </DESC> + */ +#include <stdio.h> +#include <curl/curl.h> + +int main(void) +{ + CURL *curl; + CURLcode res; + CURLSH *share; + int i; + + share = curl_share_init(); + curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); + + /* Loop the transfer and cleanup the handle properly every lap. This will + still reuse connections since the pool is in the shared object! */ + + for(i = 0; i < 3; i++) { + curl = curl_easy_init(); + if(curl) { + curl_easy_setopt(curl, CURLOPT_URL, "http://example.com"); + /* example.com is redirected, so we tell libcurl to follow redirection */ + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + /* use the connection pool in the share object */ + curl_easy_setopt(curl, CURLOPT_SHARE, share); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + + /* Perform the request, res will get the return code */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) + fprintf(stderr, "curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + + /* always cleanup */ + curl_easy_cleanup(curl); + } + } + + curl_share_cleanup(share); + return 0; +} diff --git a/lib/conncache.c b/lib/conncache.c index c79d22764..f8ef2e88b 100644 --- a/lib/conncache.c +++ b/lib/conncache.c @@ -31,11 +31,21 @@ #include "multiif.h" #include "sendf.h" #include "conncache.h" +#include "share.h" +#include "sigpipe.h" +#include "connect.h" + /* The last 3 #include files should be in this order */ #include "curl_printf.h" #include "curl_memory.h" #include "memdebug.h" +#define CONN_LOCK(x) if((x)->share) \ + Curl_share_lock((x), CURL_LOCK_DATA_CONNECT, CURL_LOCK_ACCESS_SINGLE) +#define CONN_UNLOCK(x) if((x)->share) \ + Curl_share_unlock((x), CURL_LOCK_DATA_CONNECT) + + static void conn_llist_dtor(void *user, void *element) { struct connectdata *data = element; @@ -109,8 +119,23 @@ static void free_bundle_hash_entry(void *freethis) int Curl_conncache_init(struct conncache *connc, int size) { - return Curl_hash_init(&connc->hash, size, Curl_hash_str, - Curl_str_key_compare, free_bundle_hash_entry); + int rc; + + /* allocate a new easy handle to use when closing cached connections */ + connc->closure_handle = curl_easy_init(); + if(!connc->closure_handle) + return 1; /* bad */ + + rc = Curl_hash_init(&connc->hash, size, Curl_hash_str, + Curl_str_key_compare, free_bundle_hash_entry); + if(rc) { + Curl_close(connc->closure_handle); + connc->closure_handle = NULL; + } + else + connc->closure_handle->state.conn_cache = connc; + + return rc; } void Curl_conncache_destroy(struct conncache *connc) @@ -149,7 +174,9 @@ struct connectbundle *Curl_conncache_find_bundle(struct connectdata *conn, if(connc) { char key[128]; hashkey(conn, key, sizeof(key)); + CONN_LOCK(conn->data); bundle = Curl_hash_pick(&connc->hash, key, strlen(key)); + CONN_UNLOCK(conn->data); } return bundle; @@ -206,7 +233,9 @@ CURLcode Curl_conncache_add_conn(struct conncache *connc, return result; hashkey(conn, key, sizeof(key)); + CONN_LOCK(data); rc = conncache_add_bundle(data->state.conn_cache, key, new_bundle); + CONN_UNLOCK(data); if(!rc) { bundle_destroy(new_bundle); @@ -215,12 +244,15 @@ CURLcode Curl_conncache_add_conn(struct conncache *connc, bundle = new_bundle; } + CONN_LOCK(data); result = bundle_add_conn(bundle, conn); if(result) { if(new_bundle) conncache_remove_bundle(data->state.conn_cache, new_bundle); + CONN_UNLOCK(data); return result; } + CONN_UNLOCK(data); conn->connection_id = connc->next_connection_id++; connc->num_connections++; @@ -240,11 +272,11 @@ void Curl_conncache_remove_conn(struct conncache *connc, /* The bundle pointer can be NULL, since this function can be called due to a failed connection attempt, before being added to a bundle */ if(bundle) { + CONN_LOCK(conn->data); bundle_remove_conn(bundle, conn); - if(bundle->num_connections == 0) { + if(bundle->num_connections == 0) conncache_remove_bundle(connc, bundle); - } - + CONN_UNLOCK(conn->data); if(connc) { connc->num_connections--; @@ -261,7 +293,8 @@ void Curl_conncache_remove_conn(struct conncache *connc, Return 0 from func() to continue the loop, return 1 to abort it. */ -void Curl_conncache_foreach(struct conncache *connc, +void Curl_conncache_foreach(struct Curl_easy *data, + struct conncache *connc, void *param, int (*func)(struct connectdata *conn, void *param)) { @@ -272,6 +305,7 @@ void Curl_conncache_foreach(struct conncache *connc, if(!connc) return; + CONN_LOCK(data); Curl_hash_start_iterate(&connc->hash, &iter); he = Curl_hash_next_element(&iter); @@ -288,14 +322,21 @@ void Curl_conncache_foreach(struct conncache *connc, struct connectdata *conn = curr->ptr; curr = curr->next; - if(1 == func(conn, param)) + if(1 == func(conn, param)) { + CONN_UNLOCK(data); return; + } } } + CONN_UNLOCK(data); } /* Return the first connection found in the cache. Used when closing all - connections */ + connections. + + NOTE: no locking is done here as this is presumably only done when cleaning + up a cache! +*/ struct connectdata * Curl_conncache_find_first_connection(struct conncache *connc) { @@ -321,6 +362,90 @@ Curl_conncache_find_first_connection(struct conncache *connc) return NULL; } +/* + * This function finds the connection in the connection + * cache that has been unused for the longest time. + * + * Returns the pointer to the oldest idle connection, or NULL if none was + * found. + */ +struct connectdata * +Curl_conncache_oldest_idle(struct Curl_easy *data) +{ + struct conncache *bc = data->state.conn_cache; + struct curl_hash_iterator iter; + struct curl_llist_element *curr; + struct curl_hash_element *he; + timediff_t highscore =- 1; + timediff_t score; + struct curltime now; + struct connectdata *conn_candidate = NULL; + struct connectbundle *bundle; + + now = Curl_now(); + + CONN_LOCK(data); + Curl_hash_start_iterate(&bc->hash, &iter); + + he = Curl_hash_next_element(&iter); + while(he) { + struct connectdata *conn; + + bundle = he->ptr; + + curr = bundle->conn_list.head; + while(curr) { + conn = curr->ptr; + + if(!conn->inuse) { + /* Set higher score for the age passed since the connection was used */ + score = Curl_timediff(now, conn->now); + + if(score > highscore) { + highscore = score; + conn_candidate = conn; + } + } + curr = curr->next; + } + + he = Curl_hash_next_element(&iter); + } + CONN_UNLOCK(data); + + return conn_candidate; +} + +void Curl_conncache_close_all_connections(struct conncache *connc) +{ + struct connectdata *conn; + + conn = Curl_conncache_find_first_connection(connc); + while(conn) { + SIGPIPE_VARIABLE(pipe_st); + conn->data = connc->closure_handle; + + sigpipe_ignore(conn->data, &pipe_st); + conn->data->easy_conn = NULL; /* clear the easy handle's connection + pointer */ + /* This will remove the connection from the cache */ + connclose(conn, "kill all"); + (void)Curl_disconnect(conn, FALSE); + sigpipe_restore(&pipe_st); + + conn = Curl_conncache_find_first_connection(connc); + } + + if(connc->closure_handle) { + SIGPIPE_VARIABLE(pipe_st); + sigpipe_ignore(connc->closure_handle, &pipe_st); + + Curl_hostcache_clean(connc->closure_handle, + connc->closure_handle->dns.hostcache); + Curl_close(connc->closure_handle); + sigpipe_restore(&pipe_st); + } +} #if 0 /* Useful for debugging the connection cache */ diff --git a/lib/conncache.h b/lib/conncache.h index 14be4e8e7..0d97a6cef 100644 --- a/lib/conncache.h +++ b/lib/conncache.h @@ -28,6 +28,8 @@ struct conncache { size_t num_connections; long next_connection_id; struct curltime last_cleanup; + /* handle used for closing cached connections */ + struct Curl_easy *closure_handle; }; #define BUNDLE_NO_MULTIUSE -1 @@ -41,8 +43,8 @@ struct connectbundle { struct curl_llist conn_list; /* The connectdata members of the bundle */ }; +/* returns 1 on error, 0 is fine */ int Curl_conncache_init(struct conncache *, int size); - void Curl_conncache_destroy(struct conncache *connc); /* return the correct bundle, to a host or a proxy */ @@ -55,7 +57,8 @@ CURLcode Curl_conncache_add_conn(struct conncache *connc, void Curl_conncache_remove_conn(struct conncache *connc, struct connectdata *conn); -void Curl_conncache_foreach(struct conncache *connc, +void Curl_conncache_foreach(struct Curl_easy *data, + struct conncache *connc, void *param, int (*func)(struct connectdata *conn, void *param)); @@ -63,6 +66,9 @@ void Curl_conncache_foreach(struct conncache *connc, struct connectdata * Curl_conncache_find_first_connection(struct conncache *connc); +struct connectdata * +Curl_conncache_oldest_idle(struct Curl_easy *data); +void Curl_conncache_close_all_connections(struct conncache *connc); void Curl_conncache_print(struct conncache *connc); #endif /* HEADER_CURL_CONNCACHE_H */ diff --git a/lib/connect.c b/lib/connect.c index 84a63aefd..28c6e9ed2 100755 --- a/lib/connect.c +++ b/lib/connect.c @@ -1224,7 +1224,7 @@ curl_socket_t Curl_getconnectinfo(struct Curl_easy *data, find.tofind = data->state.lastconnect; find.found = FALSE; - Curl_conncache_foreach(data->multi_easy? + Curl_conncache_foreach(data, data->multi_easy? &data->multi_easy->conn_cache: &data->multi->conn_cache, &find, conn_is_conn); diff --git a/lib/multi.c b/lib/multi.c index 262f192ca..9728e5a2f 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -326,14 +326,6 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ Curl_llist_init(&multi->msglist, multi_freeamsg); Curl_llist_init(&multi->pending, multi_freeamsg); - /* allocate a new easy handle to use when closing cached connections */ - multi->closure_handle = curl_easy_init(); - if(!multi->closure_handle) - goto error; - - multi->closure_handle->multi = multi; - multi->closure_handle->state.conn_cache = &multi->conn_cache; - multi->max_pipeline_length = 5; /* -1 means it not set by user, use the default value */ @@ -345,8 +337,6 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ Curl_hash_destroy(&multi->sockhash); Curl_hash_destroy(&multi->hostcache); Curl_conncache_destroy(&multi->conn_cache); - Curl_close(multi->closure_handle); - multi->closure_handle = NULL; Curl_llist_destroy(&multi->msglist, NULL); Curl_llist_destroy(&multi->pending, NULL); @@ -407,8 +397,11 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, data->dns.hostcachetype = HCACHE_MULTI; } - /* Point to the multi's connection cache */ - data->state.conn_cache = &multi->conn_cache; + /* Point to the shared or multi handle connection cache */ + if(data->share && (data->share->specifier & (1<< CURL_LOCK_DATA_CONNECT))) + data->state.conn_cache = &data->share->conn_cache; + else + data->state.conn_cache = &multi->conn_cache; /* This adds the new entry at the 'end' of the doubly-linked circular list of Curl_easy structs to try and maintain a FIFO queue so @@ -462,8 +455,8 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, state somewhat we clone the timeouts from each added handle so that the closure handle always has the same timeouts as the most recently added easy handle. */ - multi->closure_handle->set.timeout = data->set.timeout; - multi->closure_handle->set.server_response_timeout = + data->state.conn_cache->closure_handle->set.timeout = data->set.timeout; + data->state.conn_cache->closure_handle->set.server_response_timeout = data->set.server_response_timeout; update_timer(multi); @@ -504,7 +497,7 @@ ConnectionDone(struct Curl_easy *data, struct connectdata *conn) data->state.conn_cache->num_connections > maxconnects) { infof(data, "Connection cache is full, closing the oldest one.\n"); - conn_candidate = Curl_oldest_idle_connection(data); + conn_candidate = Curl_conncache_oldest_idle(data); if(conn_candidate) { /* Set the connection's owner correctly */ @@ -2201,36 +2194,12 @@ CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles) return returncode; } -static void close_all_connections(struct Curl_multi *multi) -{ - struct connectdata *conn; - - conn = Curl_conncache_find_first_connection(&multi->conn_cache); - while(conn) { - SIGPIPE_VARIABLE(pipe_st); - conn->data = multi->closure_handle; - - sigpipe_ignore(conn->data, &pipe_st); - conn->data->easy_conn = NULL; /* clear the easy handle's connection - pointer */ - /* This will remove the connection from the cache */ - connclose(conn, "kill all"); - (void)Curl_disconnect(conn, FALSE); - sigpipe_restore(&pipe_st); - - conn = Curl_conncache_find_first_connection(&multi->conn_cache); - } -} - CURLMcode curl_multi_cleanup(struct Curl_multi *multi) { struct Curl_easy *data; struct Curl_easy *nextdata; if(GOOD_MULTI_HANDLE(multi)) { - bool restore_pipe = FALSE; - SIGPIPE_VARIABLE(pipe_st); - multi->type = 0; /* not good anymore */ /* Firsrt remove all remaining easy handles */ @@ -2255,18 +2224,7 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) } /* Close all the connections in the connection cache */ - close_all_connections(multi); - - if(multi->closure_handle) { - sigpipe_ignore(multi->closure_handle, &pipe_st); - restore_pipe = TRUE; - - multi->closure_handle->dns.hostcache = &multi->hostcache; - Curl_hostcache_clean(multi->closure_handle, - multi->closure_handle->dns.hostcache); - - Curl_close(multi->closure_handle); - } + Curl_conncache_close_all_connections(&multi->conn_cache); Curl_hash_destroy(&multi->sockhash); Curl_conncache_destroy(&multi->conn_cache); @@ -2280,8 +2238,6 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) Curl_pipeline_set_server_blacklist(NULL, &multi->pipelining_server_bl); free(multi); - if(restore_pipe) - sigpipe_restore(&pipe_st); return CURLM_OK; } diff --git a/lib/multihandle.h b/lib/multihandle.h index 405753947..de9a7cf59 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -114,10 +114,6 @@ struct Curl_multi { /* Shared connection cache (bundles)*/ struct conncache conn_cache; - /* This handle will be used for closing the cached connections in - curl_multi_cleanup() */ - struct Curl_easy *closure_handle; - long maxconnects; /* if >0, a fixed limit of the maximum number of entries we're allowed to grow the connection cache to */ diff --git a/lib/share.c b/lib/share.c index 5b3957fcf..c1ce1aab1 100644 --- a/lib/share.c +++ b/lib/share.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -102,6 +102,8 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) break; case CURL_LOCK_DATA_CONNECT: /* not supported (yet) */ + if(Curl_conncache_init(&share->conn_cache, 103)) + return CURLSHE_NOMEM; break; default: @@ -186,6 +188,8 @@ curl_share_cleanup(struct Curl_share *share) return CURLSHE_IN_USE; } + Curl_conncache_close_all_connections(&share->conn_cache); + Curl_conncache_destroy(&share->conn_cache); Curl_hash_destroy(&share->hostcache); #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) diff --git a/lib/share.h b/lib/share.h index c039a16cb..4b13406d9 100644 --- a/lib/share.h +++ b/lib/share.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -26,6 +26,7 @@ #include <curl/curl.h> #include "cookie.h" #include "urldata.h" +#include "conncache.h" /* SalfordC says "A structure member may not be volatile". Hence: */ @@ -43,7 +44,7 @@ struct Curl_share { curl_lock_function lockfunc; curl_unlock_function unlockfunc; void *clientdata; - + struct conncache conn_cache; struct curl_hash hostcache; #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) struct CookieInfo *cookies; @@ -3448,58 +3448,6 @@ static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke) } } -/* - * This function finds the connection in the connection - * cache that has been unused for the longest time. - * - * Returns the pointer to the oldest idle connection, or NULL if none was - * found. - */ -struct connectdata * -Curl_oldest_idle_connection(struct Curl_easy *data) -{ - struct conncache *bc = data->state.conn_cache; - struct curl_hash_iterator iter; - struct curl_llist_element *curr; - struct curl_hash_element *he; - timediff_t highscore =- 1; - timediff_t score; - struct curltime now; - struct connectdata *conn_candidate = NULL; - struct connectbundle *bundle; - - now = Curl_now(); - - Curl_hash_start_iterate(&bc->hash, &iter); - - he = Curl_hash_next_element(&iter); - while(he) { - struct connectdata *conn; - - bundle = he->ptr; - - curr = bundle->conn_list.head; - while(curr) { - conn = curr->ptr; - - if(!conn->inuse) { - /* Set higher score for the age passed since the connection was used */ - score = Curl_timediff(now, conn->now); - - if(score > highscore) { - highscore = score; - conn_candidate = conn; - } - } - curr = curr->next; - } - - he = Curl_hash_next_element(&iter); - } - - return conn_candidate; -} - static bool proxy_info_matches(const struct proxy_info* data, const struct proxy_info* needle) @@ -3619,7 +3567,7 @@ static void prune_dead_connections(struct Curl_easy *data) time_t elapsed = Curl_timediff(now, data->state.conn_cache->last_cleanup); if(elapsed >= 1000L) { - Curl_conncache_foreach(data->state.conn_cache, data, + Curl_conncache_foreach(data, data->state.conn_cache, data, call_disconnect_if_dead); data->state.conn_cache->last_cleanup = now; } @@ -6996,7 +6944,7 @@ static CURLcode create_conn(struct Curl_easy *data, struct connectdata *conn_candidate; /* The cache is full. Let's see if we can kill a connection. */ - conn_candidate = Curl_oldest_idle_connection(data); + conn_candidate = Curl_conncache_oldest_idle(data); if(conn_candidate) { /* Set the connection's owner correctly, then kill it */ @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al. + * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -57,8 +57,6 @@ CURLcode Curl_addHandleToPipeline(struct Curl_easy *handle, struct curl_llist *pipeline); int Curl_removeHandleFromPipeline(struct Curl_easy *handle, struct curl_llist *pipeline); -struct connectdata * -Curl_oldest_idle_connection(struct Curl_easy *data); /* remove the specified connection from all (possible) pipelines and related queues */ void Curl_getoff_all_pipelines(struct Curl_easy *data, |