1 files changed, 208 insertions, 60 deletions

diff --git a/lib/url.c b/lib/url.c
index cbe0f4659..a14c0626b 100644
--- a/lib/url.c
+++ b/lib/url.c
@@ -123,6 +123,7 @@ int curl_win32_idn_to_ascii(const char *in, char **out);
 #include "bundles.h"
 #include "conncache.h"
 #include "multihandle.h"
+#include "pipeline.h"
 
 #define _MPRINTF_REPLACE /* use our functions only */
 #include <curl/mprintf.h>
@@ -134,6 +135,9 @@ int curl_win32_idn_to_ascii(const char *in, char **out);
 /* Local static prototypes */
 static struct connectdata *
 find_oldest_idle_connection(struct SessionHandle *data);
+static struct connectdata *
+find_oldest_idle_connection_in_bundle(struct SessionHandle *data,
+                                      struct connectbundle *bundle);
 static void conn_free(struct connectdata *conn);
 static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke);
 static CURLcode do_init(struct connectdata *conn);
@@ -2470,13 +2474,9 @@ static void conn_free(struct connectdata *conn)
 
   Curl_llist_destroy(conn->send_pipe, NULL);
   Curl_llist_destroy(conn->recv_pipe, NULL);
-  Curl_llist_destroy(conn->pend_pipe, NULL);
-  Curl_llist_destroy(conn->done_pipe, NULL);
 
   conn->send_pipe = NULL;
   conn->recv_pipe = NULL;
-  conn->pend_pipe = NULL;
-  conn->done_pipe = NULL;
 
   Curl_safefree(conn->localdev);
   Curl_free_ssl_config(&conn->ssl_config);
@@ -2566,11 +2566,9 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection)
   Curl_ssl_close(conn, FIRSTSOCKET);
 
   /* Indicate to all handles on the pipe that we're dead */
-  if(Curl_isPipeliningEnabled(data)) {
+  if(Curl_multi_pipeline_enabled(data->multi)) {
     signalPipeClose(conn->send_pipe, TRUE);
     signalPipeClose(conn->recv_pipe, TRUE);
-    signalPipeClose(conn->pend_pipe, TRUE);
-    signalPipeClose(conn->done_pipe, FALSE);
   }
 
   conn_free(conn);
@@ -2602,7 +2600,7 @@ static bool IsPipeliningPossible(const struct SessionHandle *handle,
                                  const struct connectdata *conn)
 {
   if((conn->handler->protocol & CURLPROTO_HTTP) &&
-     handle->multi && Curl_multi_canPipeline(handle->multi) &&
+     Curl_multi_pipeline_enabled(handle->multi) &&
      (handle->set.httpreq == HTTPREQ_GET ||
       handle->set.httpreq == HTTPREQ_HEAD) &&
      handle->set.httpversion != CURL_HTTP_VERSION_1_0)
@@ -2613,10 +2611,7 @@ static bool IsPipeliningPossible(const struct SessionHandle *handle,
 
 bool Curl_isPipeliningEnabled(const struct SessionHandle *handle)
 {
-  if(handle->multi && Curl_multi_canPipeline(handle->multi))
-    return TRUE;
-
-  return FALSE;
+  return Curl_multi_pipeline_enabled(handle->multi);
 }
 
 CURLcode Curl_addHandleToPipeline(struct SessionHandle *data,
@@ -2624,6 +2619,7 @@ CURLcode Curl_addHandleToPipeline(struct SessionHandle *data,
 {
   if(!Curl_llist_insert_next(pipeline, pipeline->tail, data))
     return CURLE_OUT_OF_MEMORY;
+  infof(data, "Curl_addHandleToPipeline: length: %d\n", pipeline->size);
   return CURLE_OK;
 }
 
@@ -2683,8 +2679,6 @@ void Curl_getoff_all_pipelines(struct SessionHandle *data,
     conn->readchannel_inuse = FALSE;
   if(Curl_removeHandleFromPipeline(data, conn->send_pipe) && send_head)
     conn->writechannel_inuse = FALSE;
-  Curl_removeHandleFromPipeline(data, conn->pend_pipe);
-  Curl_removeHandleFromPipeline(data, conn->done_pipe);
 }
 
 static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke)
@@ -2715,8 +2709,8 @@ static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke)
 }
 
 /*
- * This function kills and removes an existing connection in the connection
- * cache. The connection that has been unused for the longest time.
+ * This function finds the connection in the connection
+ * cache that has been unused for the longest time.
  *
  * Returns the pointer to the oldest idle connection, or NULL if none was
  * found.
@@ -2767,6 +2761,47 @@ find_oldest_idle_connection(struct SessionHandle *data)
 }
 
 /*
+ * This function finds the connection in the connection
+ * bundle that has been unused for the longest time.
+ *
+ * Returns the pointer to the oldest idle connection, or NULL if none was
+ * found.
+ */
+static struct connectdata *
+find_oldest_idle_connection_in_bundle(struct SessionHandle *data,
+                                      struct connectbundle *bundle)
+{
+  struct curl_llist_element *curr;
+  long highscore=-1;
+  long score;
+  struct timeval now;
+  struct connectdata *conn_candidate = NULL;
+  struct connectdata *conn;
+
+  (void)data;
+
+  now = Curl_tvnow();
+
+  curr = bundle->conn_list->head;
+  while(curr) {
+    conn = curr->ptr;
+
+    if(!conn->inuse) {
+      /* Set higher score for the age passed since the connection was used */
+      score = Curl_tvdiff(now, conn->now);
+
+      if(score > highscore) {
+        highscore = score;
+        conn_candidate = conn;
+      }
+    }
+    curr = curr->next;
+  }
+
+  return conn_candidate;
+}
+
+/*
  * Given one filled in connection struct (named needle), this function should
  * detect if there already is one that has all the significant details
  * exactly the same and thus should be used instead.
@@ -2774,11 +2809,15 @@ find_oldest_idle_connection(struct SessionHandle *data)
  * If there is a match, this function returns TRUE - and has marked the
  * connection as 'in-use'. It must later be called with ConnectionDone() to
  * return back to 'idle' (unused) state.
+ *
+ * The force_reuse flag is set if the connection must be used, even if
+ * the pipelining strategy wants to open a new connection instead of reusing.
  */
 static bool
 ConnectionExists(struct SessionHandle *data,
                  struct connectdata *needle,
-                 struct connectdata **usethis)
+                 struct connectdata **usethis,
+                 bool *force_reuse)
 {
   struct connectdata *check;
   struct connectdata *chosen = 0;
@@ -2787,15 +2826,30 @@ ConnectionExists(struct SessionHandle *data,
                   (data->state.authhost.want==CURLAUTH_NTLM_WB) ? TRUE : FALSE;
   struct connectbundle *bundle;
 
+  *force_reuse = FALSE;
+
+  /* We can't pipe if the site is blacklisted */
+  if(canPipeline && Curl_pipeline_site_blacklisted(data, needle)) {
+    canPipeline = FALSE;
+  }
+
   /* Look up the bundle with all the connections to this
      particular host */
   bundle = Curl_conncache_find_bundle(data->state.conn_cache,
                                       needle->host.name);
   if(bundle) {
+    size_t max_pipe_len = Curl_multi_max_pipeline_length(data->multi);
+    size_t best_pipe_len = max_pipe_len;
     struct curl_llist_element *curr;
 
     infof(data, "Found bundle for host %s: %p\n", needle->host.name, bundle);
 
+    /* We can't pipe if we don't know anything about the server */
+    if(canPipeline && !bundle->server_supports_pipelining) {
+      infof(data, "Server doesn't support pipelining\n");
+      canPipeline = FALSE;
+    }
+
     curr = bundle->conn_list->head;
     while(curr) {
       bool match = FALSE;
@@ -2845,12 +2899,6 @@ ConnectionExists(struct SessionHandle *data,
           if(!IsPipeliningPossible(rh, check))
             continue;
         }
-#ifdef DEBUGBUILD
-      if(pipeLen > MAX_PIPELINE_LENGTH) {
-        infof(data, "BAD! Connection #%ld has too big pipeline!\n",
-              check->connection_id);
-      }
-#endif
       }
       else {
         if(pipeLen > 0) {
@@ -2989,26 +3037,60 @@ ConnectionExists(struct SessionHandle *data,
       }
 
       if(match) {
-        chosen = check;
+        /* If we are looking for an NTLM connection, check if this is already
+           authenticating with the right credentials. If not, keep looking so
+           that we can reuse NTLM connections if possible. (Especially we
+           must not reuse the same connection if partway through
+           a handshake!) */
+        if(wantNTLM) {
+          if(credentialsMatch && check->ntlm.state != NTLMSTATE_NONE) {
+            chosen = check;
+
+            /* We must use this connection, no other */
+            *force_reuse = TRUE;
+            break;
+          }
+          else
+            continue;
+        }
 
-        /* If we are not looking for an NTLM connection, we can choose this one
-           immediately. */
-        if(!wantNTLM)
-          break;
+        if(canPipeline) {
+          /* We can pipeline if we want to. Let's continue looking for
+             the optimal connection to use, i.e the shortest pipe that is not
+             blacklisted. */
 
-        /* Otherwise, check if this is already authenticating with the right
-           credentials. If not, keep looking so that we can reuse NTLM
-           connections if possible. (Especially we must reuse the same
-           connection if partway through a handshake!) */
-        if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE)
+          if(pipeLen == 0) {
+            /* We have the optimal connection. Let's stop looking. */
+            chosen = check;
+            break;
+          }
+
+          /* We can't use the connection if the pipe is full */
+          if(pipeLen >= max_pipe_len)
+            continue;
+
+          /* We can't use the connection if the pipe is penalized */
+          if(Curl_pipeline_penalized(data, check))
+            continue;
+
+          if(pipeLen < best_pipe_len) {
+            /* This connection has a shorter pipe so far. We'll pick this
+               and continue searching */
+            chosen = check;
+            best_pipe_len = pipeLen;
+            continue;
+          }
+        }
+        else {
+          /* We have found a connection. Let's stop searching. */
+          chosen = check;
           break;
+        }
       }
     }
   }
 
   if(chosen) {
-    chosen->inuse = TRUE; /* mark this as being in use so that no other
-                            handle in a multi stack may nick it */
     *usethis = chosen;
     return TRUE; /* yes, we found one to use! */
   }
@@ -3475,7 +3557,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data)
   conn->response_header = NULL;
 #endif
 
-  if(data->multi && Curl_multi_canPipeline(data->multi) &&
+  if(Curl_multi_pipeline_enabled(data->multi) &&
       !conn->master_buffer) {
     /* Allocate master_buffer to be used for pipelining */
     conn->master_buffer = calloc(BUFSIZE, sizeof (char));
@@ -3486,10 +3568,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data)
   /* Initialize the pipeline lists */
   conn->send_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor);
   conn->recv_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor);
-  conn->pend_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor);
-  conn->done_pipe = Curl_llist_alloc((curl_llist_dtor) llist_dtor);
-  if(!conn->send_pipe || !conn->recv_pipe || !conn->pend_pipe ||
-     !conn->done_pipe)
+  if(!conn->send_pipe || !conn->recv_pipe)
     goto error;
 
 #if defined(HAVE_KRB4) || defined(HAVE_GSSAPI)
@@ -3515,13 +3594,9 @@ static struct connectdata *allocate_conn(struct SessionHandle *data)
 
   Curl_llist_destroy(conn->send_pipe, NULL);
   Curl_llist_destroy(conn->recv_pipe, NULL);
-  Curl_llist_destroy(conn->pend_pipe, NULL);
-  Curl_llist_destroy(conn->done_pipe, NULL);
 
   conn->send_pipe = NULL;
   conn->recv_pipe = NULL;
-  conn->pend_pipe = NULL;
-  conn->done_pipe = NULL;
 
   Curl_safefree(conn->master_buffer);
   Curl_safefree(conn->localdev);
@@ -4623,13 +4698,9 @@ static void reuse_conn(struct connectdata *old_conn,
 
   Curl_llist_destroy(old_conn->send_pipe, NULL);
   Curl_llist_destroy(old_conn->recv_pipe, NULL);
-  Curl_llist_destroy(old_conn->pend_pipe, NULL);
-  Curl_llist_destroy(old_conn->done_pipe, NULL);
 
   old_conn->send_pipe = NULL;
   old_conn->recv_pipe = NULL;
-  old_conn->pend_pipe = NULL;
-  old_conn->done_pipe = NULL;
 
   Curl_safefree(old_conn->master_buffer);
 }
@@ -4663,6 +4734,10 @@ static CURLcode create_conn(struct SessionHandle *data,
   bool reuse;
   char *proxy = NULL;
   bool prot_missing = FALSE;
+  bool no_connections_available = FALSE;
+  bool force_reuse;
+  size_t max_host_connections = Curl_multi_max_host_connections(data->multi);
+  size_t max_total_connections = Curl_multi_max_total_connections(data->multi);
 
   *async = FALSE;
 
@@ -4963,7 +5038,25 @@ static CURLcode create_conn(struct SessionHandle *data,
   if(data->set.reuse_fresh && !data->state.this_is_a_follow)
     reuse = FALSE;
   else
-    reuse = ConnectionExists(data, conn, &conn_temp);
+    reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse);
+
+  /* If we found a reusable connection, we may still want to
+     open a new connection if we are pipelining. */
+  if(reuse && !force_reuse && IsPipeliningPossible(data, conn_temp)) {
+    size_t pipelen = conn_temp->send_pipe->size + conn_temp->recv_pipe->size;
+    if(pipelen > 0) {
+      infof(data, "Found connection %d, with requests in the pipe (%d)\n",
+            conn_temp->connection_id, pipelen);
+
+      if(conn_temp->bundle->num_connections < max_host_connections &&
+         data->state.conn_cache->num_connections < max_total_connections) {
+        /* We want a new connection anyway */
+        reuse = FALSE;
+
+        infof(data, "We can reuse, but we want a new connection anyway\n");
+      }
+    }
+  }
 
   if(reuse) {
     /*
@@ -4972,6 +5065,8 @@ static CURLcode create_conn(struct SessionHandle *data,
      * just allocated before we can move along and use the previously
      * existing one.
      */
+    conn_temp->inuse = TRUE; /* mark this as being in use so that no other
+                                handle in a multi stack may nick it */
     reuse_conn(conn, conn_temp);
     free(conn);          /* we don't need this anymore */
     conn = conn_temp;
@@ -4985,14 +5080,66 @@ static CURLcode create_conn(struct SessionHandle *data,
           conn->proxy.name?conn->proxy.dispname:conn->host.dispname);
   }
   else {
-    /*
-     * This is a brand new connection, so let's store it in the connection
-     * cache of ours!
-     */
-    conn->inuse = TRUE;
-    ConnectionStore(data, conn);
+    /* We have decided that we want a new connection. However, we may not
+       be able to do that if we have reached the limit of how many
+       connections we are allowed to open. */
+    struct connectbundle *bundle;
+
+    bundle = Curl_conncache_find_bundle(data->state.conn_cache,
+                                        conn->host.name);
+    if(max_host_connections > 0 && bundle &&
+       (bundle->num_connections >= max_host_connections)) {
+      struct connectdata *conn_candidate;
+
+      /* The bundle is full. Let's see if we can kill a connection. */
+      conn_candidate = find_oldest_idle_connection_in_bundle(data, bundle);
+
+      if(conn_candidate) {
+        /* Set the connection's owner correctly, then kill it */
+        conn_candidate->data = data;
+        (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE);
+      }
+      else
+        no_connections_available = TRUE;
+    }
+
+    if(max_total_connections > 0 &&
+       (data->state.conn_cache->num_connections >= max_total_connections)) {
+      struct connectdata *conn_candidate;
+
+      /* The cache is full. Let's see if we can kill a connection. */
+      conn_candidate = find_oldest_idle_connection(data);
+
+      if(conn_candidate) {
+        /* Set the connection's owner correctly, then kill it */
+        conn_candidate->data = data;
+        (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE);
+      }
+      else
+        no_connections_available = TRUE;
+    }
+
+
+    if(no_connections_available) {
+      infof(data, "No connections available.\n");
+
+      conn_free(conn);
+      *in_connect = NULL;
+
+      return CURLE_NO_CONNECTION_AVAILABLE;
+    }
+    else {
+      /*
+       * This is a brand new connection, so let's store it in the connection
+       * cache of ours!
+       */
+      ConnectionStore(data, conn);
+    }
   }
 
+  /* Mark the connection as used */
+  conn->inuse = TRUE;
+
   /* Setup and init stuff before DO starts, in preparing for the transfer. */
   do_init(conn);
 
@@ -5167,6 +5314,11 @@ CURLcode Curl_connect(struct SessionHandle *data,
     }
   }
 
+  if(code == CURLE_NO_CONNECTION_AVAILABLE) {
+    *in_connect = NULL;
+    return code;
+  }
+
   if(code && *in_connect) {
     /* We're not allowed to return failure with memory left allocated
        in the connectdata struct, free those here */
@@ -5258,12 +5410,8 @@ CURLcode Curl_done(struct connectdata **connp,
      state it is for re-using, so we're forced to close it. In a perfect world
      we can add code that keep track of if we really must close it here or not,
      but currently we have no such detail knowledge.
-
-     connection_id == -1 here means that the connection has not been added
-     to the connection cache (OOM) and thus we must disconnect it here.
   */
-  if(data->set.reuse_forbid || conn->bits.close || premature ||
-     (-1 == conn->connection_id)) {
+  if(data->set.reuse_forbid || conn->bits.close || premature) {
     CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */
 
     /* If we had an error already, make sure we return that one. But