Introducing a new persistent connection caching system using "bundles".

A bundle is a list of all persistent connections to the same host.
The connection cache consists of a hash of bundles, with the
hostname as the key.
The benefits may not be obvious, but they are two:

1) Faster search for connections to reuse, since the hash
   lookup only finds connections to the host in question.
2) It lays out the groundworks for an upcoming patch,
   which will introduce multiple HTTP pipelines.

This patch also removes the awkward list of "closure handles",
which were needed to send QUIT commands to the FTP server
when closing a connection.
Now we allocate a separate closure handle and use that
one to close all connections.

This has been tested in a live system for a few weeks, and of
course passes the test suite.

1 files changed, 66 insertions, 247 deletions

diff --git a/lib/multi.c b/lib/multi.c
index d2d154fa7..e309c0d96 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -44,6 +44,8 @@
 #include "select.h"
 #include "warnless.h"
 #include "speedcheck.h"
+#include "conncache.h"
+#include "bundles.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -97,11 +99,6 @@ typedef enum {
 #define GETSOCK_READABLE (0x00ff)
 #define GETSOCK_WRITABLE (0xff00)
 
-struct closure {
-  struct closure *next; /* a simple one-way list of structs */
-  struct SessionHandle *easy_handle;
-};
-
 struct Curl_one_easy {
   /* first, two fields for the linked list of these */
   struct Curl_one_easy *next;
@@ -164,14 +161,16 @@ struct Curl_multi {
   /* Whether pipelining is enabled for this multi handle */
   bool pipelining_enabled;
 
-  /* shared connection cache */
-  struct conncache *connc;
+  /* Shared connection cache (bundles)*/
+  struct conncache *conn_cache;
+
+  /* This handle will be used for closing the cached connections in
+     curl_multi_cleanup() */
+  struct SessionHandle *closure_handle;
+
   long maxconnects; /* if >0, a fixed limit of the maximum number of entries
                        we're allowed to grow the connection cache to */
 
-  /* list of easy handles kept around for doing nice connection closures */
-  struct closure *closure;
-
   /* timer callback and user data pointer for the *socket() API */
   curl_multi_timer_callback timer_cb;
   void *timer_userp;
@@ -179,12 +178,8 @@ struct Curl_multi {
                                     previous callback */
 };
 
-static void multi_connc_remove_handle(struct Curl_multi *multi,
-                                      struct SessionHandle *data);
 static void singlesocket(struct Curl_multi *multi,
                          struct Curl_one_easy *easy);
-static CURLMcode add_closure(struct Curl_multi *multi,
-                             struct SessionHandle *data);
 static int update_timer(struct Curl_multi *multi);
 
 static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle,
@@ -228,7 +223,7 @@ static void multi_freetimeout(void *a, void *b);
 static void multistate(struct Curl_one_easy *easy, CURLMstate state)
 {
 #ifdef DEBUGBUILD
-  long connectindex = -5000;
+  long connection_id = -5000;
 #endif
   CURLMstate oldstate = easy->state;
 
@@ -242,12 +237,12 @@ static void multistate(struct Curl_one_easy *easy, CURLMstate state)
   if(easy->easy_conn) {
     if(easy->state > CURLM_STATE_CONNECT &&
        easy->state < CURLM_STATE_COMPLETED)
-      connectindex = easy->easy_conn->connectindex;
+      connection_id = easy->easy_conn->connection_id;
 
     infof(easy->easy_handle,
           "STATE: %s => %s handle %p; (connection #%ld) \n",
           statename[oldstate], statename[easy->state],
-          (char *)easy, connectindex);
+          (char *)easy, connection_id);
   }
 #endif
   if(state == CURLM_STATE_COMPLETED)
@@ -391,7 +386,6 @@ static void multi_freeamsg(void *a, void *b)
   (void)b;
 }
 
-
 CURLM *curl_multi_init(void)
 {
   struct Curl_multi *multi = calloc(1, sizeof(struct Curl_multi));
@@ -409,8 +403,8 @@ CURLM *curl_multi_init(void)
   if(!multi->sockhash)
     goto error;
 
-  multi->connc = Curl_mk_connc(CONNCACHE_MULTI, -1L);
-  if(!multi->connc)
+  multi->conn_cache = Curl_conncache_init(CONNCACHE_MULTI);
+  if(!multi->conn_cache)
     goto error;
 
   multi->msglist = Curl_llist_alloc(multi_freeamsg);
@@ -431,8 +425,8 @@ CURLM *curl_multi_init(void)
   multi->sockhash = NULL;
   Curl_hash_destroy(multi->hostcache);
   multi->hostcache = NULL;
-  Curl_rm_connc(multi->connc);
-  multi->connc = NULL;
+  Curl_conncache_destroy(multi->conn_cache);
+  multi->conn_cache = NULL;
 
   free(multi);
   return NULL;
@@ -443,8 +437,6 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
 {
   struct curl_llist *timeoutlist;
   struct Curl_one_easy *easy;
-  struct closure *cl;
-  struct closure *prev = NULL;
   struct Curl_multi *multi = (struct Curl_multi *)multi_handle;
   struct SessionHandle *data = (struct SessionHandle *)easy_handle;
 
@@ -462,22 +454,13 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
     /* possibly we should create a new unique error code for this condition */
     return CURLM_BAD_EASY_HANDLE;
 
-  /* We want the connection cache to have plenty of room. Before we supported
-     the shared cache every single easy handle had 5 entries in their cache
-     by default. */
-  if(((multi->num_easy + 1) * 4) > multi->connc->num) {
-    long newmax = (multi->num_easy + 1) * 4;
-
-    if(multi->maxconnects && (newmax > multi->maxconnects))
-      /* don't grow beyond the allowed size */
-      newmax = multi->maxconnects;
-
-    if(newmax > multi->connc->num) {
-      /* we only do this is we can in fact grow the cache */
-      CURLcode res = Curl_ch_connc(data, multi->connc, newmax);
-      if(res)
-        return CURLM_OUT_OF_MEMORY;
-    }
+  /* This is a good time to allocate a fresh easy handle to use when closing
+     cached connections */
+  if(!multi->closure_handle) {
+    multi->closure_handle =
+      (struct SessionHandle *)curl_easy_init();
+    Curl_easy_addmulti(easy_handle, multi_handle);
+    multi->closure_handle->state.conn_cache = multi->conn_cache;
   }
 
   /* Allocate and initialize timeout list for easy handle */
@@ -504,27 +487,6 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
   data->state.timeoutlist = timeoutlist;
   timeoutlist = NULL;
 
-  /* Remove handle from the list of 'closure handles' in case it is there */
-  cl = multi->closure;
-  while(cl) {
-    struct closure *next = cl->next;
-    if(cl->easy_handle == data) {
-      /* Remove node from list */
-      free(cl);
-      if(prev)
-        prev->next = next;
-      else
-        multi->closure = next;
-      /* removed from closure list now, this might reuse an existing
-         existing connection but we don't know that at this point */
-      data->state.shared_conn = NULL;
-      /* No need to continue, handle can only be present once in the list */
-      break;
-    }
-    prev = cl;
-    cl = next;
-  }
-
   /* set the easy handle */
   easy->easy_handle = data;
   multistate(easy, CURLM_STATE_INIT);
@@ -548,16 +510,15 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle,
   }
 
   /* On a multi stack the connection cache, owned by the multi handle,
-     is shared between all easy handles within the multi handle. */
-  if(easy->easy_handle->state.connc &&
-     (easy->easy_handle->state.connc->type == CONNCACHE_PRIVATE)) {
-    /* kill old private connection cache */
-    Curl_rm_connc(easy->easy_handle->state.connc);
-    easy->easy_handle->state.connc = NULL;
+     is shared between all easy handles within the multi handle.
+     Therefore we free the private connection cache if there is one */
+  if(easy->easy_handle->state.conn_cache &&
+     easy->easy_handle->state.conn_cache->type == CONNCACHE_PRIVATE) {
+    Curl_conncache_destroy(easy->easy_handle->state.conn_cache);
   }
+
   /* Point now to this multi's connection cache */
-  easy->easy_handle->state.connc = multi->connc;
-  easy->easy_handle->state.connc->type = CONNCACHE_MULTI;
+  easy->easy_handle->state.conn_cache = multi->conn_cache;
 
   /* This adds the new entry at the 'end' of the doubly-linked circular
      list of Curl_one_easy structs to try and maintain a FIFO queue so
@@ -701,42 +662,17 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle,
            Note that this ignores the return code simply because there's
            nothing really useful to do with it anyway! */
         (void)Curl_done(&easy->easy_conn, easy->result, premature);
-
-        if(easy->easy_conn)
-          /* the connection is still alive, set back the association to enable
-             the check below to trigger TRUE */
-          easy->easy_conn->data = easy->easy_handle;
       }
       else
         /* Clear connection pipelines, if Curl_done above was not called */
         Curl_getoff_all_pipelines(easy->easy_handle, easy->easy_conn);
     }
 
-    /* figure out if the easy handle is used by one or more connections in the
-       cache */
-    multi_connc_remove_handle(multi, easy->easy_handle);
-
-    if(easy->easy_handle->state.connc->type == CONNCACHE_MULTI) {
+    if(easy->easy_handle->state.conn_cache->type == CONNCACHE_MULTI) {
       /* if this was using the shared connection cache we clear the pointer
          to that since we're not part of that handle anymore */
-      easy->easy_handle->state.connc = NULL;
-
-      /* Since we return the connection back to the communal connection pool
-         we mark the last connection as inaccessible */
-      easy->easy_handle->state.lastconnect = -1;
-
-      /* Modify the connectindex since this handle can't point to the
-         connection cache anymore.
-
-         TODO: consider if this is really what we want. The connection cache
-         is within the multi handle and that owns the connections so we should
-         not need to touch connections like this when we just remove an easy
-         handle...
-      */
-      if(easy->easy_conn && easy_owns_conn &&
-         (easy->easy_conn->send_pipe->size +
-          easy->easy_conn->recv_pipe->size == 0))
-        easy->easy_conn->connectindex = -1;
+      easy->easy_handle->state.conn_cache = NULL;
+      easy->easy_handle->state.lastconnect = NULL;
     }
 
     /* change state without using multistate(), only to make singlesocket() do
@@ -745,6 +681,12 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle,
     singlesocket(multi, easy); /* to let the application know what sockets
                                   that vanish with this handle */
 
+    /* Remove the association between the connection and the handle */
+    if(easy->easy_conn) {
+      easy->easy_conn->data = NULL;
+      easy->easy_conn = NULL;
+    }
+
     Curl_easy_addmulti(easy->easy_handle, NULL); /* clear the association
                                                     to this multi handle */
 
@@ -1324,8 +1266,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
     case CURLM_STATE_WAITDO:
       /* Wait for our turn to DO when we're pipelining requests */
 #ifdef DEBUGBUILD
-      infof(data, "Conn %ld send pipe %zu inuse %d athead %d\n",
-            easy->easy_conn->connectindex,
+      infof(data, "WAITDO: Conn %ld send pipe %zu inuse %d athead %d\n",
+            easy->easy_conn->connection_id,
             easy->easy_conn->send_pipe->size,
             easy->easy_conn->writechannel_inuse?1:0,
             isHandleAtHead(data,
@@ -1514,8 +1456,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
       }
 #ifdef DEBUGBUILD
       else {
-        infof(data, "Conn %ld recv pipe %zu inuse %d athead %d\n",
-              easy->easy_conn->connectindex,
+        infof(data, "WAITPERFORM: Conn %ld recv pipe %zu inuse %d athead %d\n",
+              easy->easy_conn->connection_id,
               easy->easy_conn->recv_pipe->size,
               easy->easy_conn->readchannel_inuse?1:0,
               isHandleAtHead(data,
@@ -1891,45 +1833,41 @@ CURLMcode curl_multi_perform(CURLM *multi_handle, int *running_handles)
   return returncode;
 }
 
+static void close_all_connections(struct Curl_multi *multi)
+{
+  struct connectdata *conn;
+
+  conn = Curl_conncache_find_first_connection(multi->conn_cache);
+  while(conn) {
+    conn->data = multi->closure_handle;
+
+    /* This will remove the connection from the cache */
+    (void)Curl_disconnect(conn, FALSE);
+
+    conn = Curl_conncache_find_first_connection(multi->conn_cache);
+  }
+}
+
 CURLMcode curl_multi_cleanup(CURLM *multi_handle)
 {
   struct Curl_multi *multi=(struct Curl_multi *)multi_handle;
   struct Curl_one_easy *easy;
   struct Curl_one_easy *nexteasy;
-  int i;
-  struct closure *cl;
-  struct closure *n;
 
   if(GOOD_MULTI_HANDLE(multi)) {
     multi->type = 0; /* not good anymore */
 
-    /* go over all connections that have close actions */
-    for(i=0; i< multi->connc->num; i++) {
-      if(multi->connc->connects[i] &&
-         multi->connc->connects[i]->handler->flags & PROTOPT_CLOSEACTION) {
-        Curl_disconnect(multi->connc->connects[i], FALSE);
-        multi->connc->connects[i] = NULL;
-      }
-    }
-    /* now walk through the list of handles we kept around only to be
-       able to close connections "properly" */
-    cl = multi->closure;
-    while(cl) {
-      cl->easy_handle->state.shared_conn = NULL; /* allow cleanup */
-      if(cl->easy_handle->state.closed)
-        /* close handle only if curl_easy_cleanup() already has been called
-           for this easy handle */
-        Curl_close(cl->easy_handle);
-      n = cl->next;
-      free(cl);
-      cl= n;
-    }
+    /* Close all the connections in the connection cache */
+    close_all_connections(multi);
+
+    Curl_close(multi->closure_handle);
+    multi->closure_handle = NULL;
 
     Curl_hash_destroy(multi->sockhash);
     multi->sockhash = NULL;
 
-    Curl_rm_connc(multi->connc);
-    multi->connc = NULL;
+    Curl_conncache_destroy(multi->conn_cache);
+    multi->conn_cache = NULL;
 
     /* remove the pending list of messages */
     Curl_llist_destroy(multi->msglist, NULL);
@@ -1947,7 +1885,7 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle)
       }
 
       /* Clear the pointer to the connection cache */
-      easy->easy_handle->state.connc = NULL;
+      easy->easy_handle->state.conn_cache = NULL;
 
       Curl_easy_addmulti(easy->easy_handle, NULL); /* clear the association */
 
@@ -2803,125 +2741,6 @@ CURLMcode curl_multi_assign(CURLM *multi_handle,
   return CURLM_OK;
 }
 
-static void multi_connc_remove_handle(struct Curl_multi *multi,
-                                      struct SessionHandle *data)
-{
-  /* a connection in the connection cache pointing to the given 'data' ? */
-  int i;
-
-  for(i=0; i< multi->connc->num; i++) {
-    struct connectdata * conn = multi->connc->connects[i];
-
-    if(conn && conn->data == data) {
-      /* If this easy_handle was the last one in charge for one or more
-         connections in the shared connection cache, we might need to keep
-         this handle around until either A) the connection is closed and
-         killed properly, or B) another easy_handle uses the connection.
-
-         The reason why we need to have a easy_handle associated with a live
-         connection is simply that some connections will need a handle to get
-         closed down properly. Currently, the only connections that need to
-         keep a easy_handle handle around are using FTP(S). Such connections
-         have the PROT_CLOSEACTION bit set.
-
-         Thus, we need to check for all connections in the shared cache that
-         points to this handle and are using PROT_CLOSEACTION. If there's any,
-         we need to add this handle to the list of "easy handles kept around
-         for nice connection closures".
-      */
-
-      if(conn->handler->flags & PROTOPT_CLOSEACTION) {
-        /* this handle is still being used by a shared connection and
-           thus we leave it around for now */
-        if(add_closure(multi, data) == CURLM_OK)
-          data->state.shared_conn = multi;
-        else {
-          /* out of memory - so much for graceful shutdown */
-          Curl_disconnect(conn, /* dead_connection */ FALSE);
-          multi->connc->connects[i] = NULL;
-          data->state.shared_conn = NULL;
-        }
-      }
-      else {
-        /* disconect the easy handle from the connection since the connection
-           will now remain but this easy handle is going */
-        data->state.shared_conn = NULL;
-        conn->data = NULL;
-      }
-    }
-  }
-}
-
-/* Add the given data pointer to the list of 'closure handles' that are kept
-   around only to be able to close some connections nicely - just make sure
-   that this handle isn't already added, like for the cases when an easy
-   handle is removed, added and removed again... */
-static CURLMcode add_closure(struct Curl_multi *multi,
-                             struct SessionHandle *data)
-{
-  struct closure *cl = multi->closure;
-  struct closure *p = NULL;
-  bool add = TRUE;
-
-  /* Before adding, scan through all the other currently kept handles and see
-     if there are any connections still referring to them and kill them if
-     not. */
-  while(cl) {
-    struct closure *n;
-    bool inuse = FALSE;
-    int i;
-
-    for(i=0; i< multi->connc->num; i++) {
-      if(multi->connc->connects[i] &&
-         (multi->connc->connects[i]->data == cl->easy_handle)) {
-        inuse = TRUE;
-        break;
-      }
-    }
-
-    n = cl->next;
-
-    if(!inuse) {
-      /* cl->easy_handle is now killable */
-
-      /* unmark it as not having a connection around that uses it anymore */
-      cl->easy_handle->state.shared_conn= NULL;
-
-      if(cl->easy_handle->state.closed) {
-        infof(data, "Delayed kill of easy handle %p\n", cl->easy_handle);
-        /* close handle only if curl_easy_cleanup() already has been called
-           for this easy handle */
-        Curl_close(cl->easy_handle);
-      }
-      if(p)
-        p->next = n;
-      else
-        multi->closure = n;
-      free(cl);
-    }
-    else {
-      if(cl->easy_handle == data)
-        add = FALSE;
-
-      p = cl;
-    }
-
-    cl = n;
-  }
-
-  if(add) {
-    cl = calloc(1, sizeof(struct closure));
-    if(!cl)
-      return CURLM_OUT_OF_MEMORY;
-
-    cl->easy_handle = data;
-    cl->next = multi->closure;
-    multi->closure = cl;
-  }
-
-  return CURLM_OK;
-}
-
 #ifdef DEBUGBUILD
 void Curl_multi_dump(const struct Curl_multi *multi_handle)
 {