Multiple pipelines and limiting the number of connections.

Introducing a number of options to the multi interface that
allows for multiple pipelines to the same host, in order to
optimize the balance between the penalty for opening new
connections and the potential pipelining latency.

Two new options for limiting the number of connections:

CURLMOPT_MAX_HOST_CONNECTIONS - Limits the number of running connections
to the same host. When adding a handle that exceeds this limit,
that handle will be put in a pending state until another handle is
finished, so we can reuse the connection.

CURLMOPT_MAX_TOTAL_CONNECTIONS - Limits the number of connections in total.
When adding a handle that exceeds this limit,
that handle will be put in a pending state until another handle is
finished. The free connection will then be reused, if possible, or
closed if the pending handle can't reuse it.

Several new options for pipelining:

CURLMOPT_MAX_PIPELINE_LENGTH - Limits the pipeling length. If a
pipeline is "full" when a connection is to be reused, a new connection
will be opened if the CURLMOPT_MAX_xxx_CONNECTIONS limits allow it.
If not, the handle will be put in a pending state until a connection is
ready (either free or a pipe got shorter).

CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE - A pipelined connection will not
be reused if it is currently processing a transfer with a content
length that is larger than this.

CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE - A pipelined connection will not
be reused if it is currently processing a chunk larger than this.

CURLMOPT_PIPELINING_SITE_BL - A blacklist of hosts that don't allow
pipelining.

CURLMOPT_PIPELINING_SERVER_BL - A blacklist of server types that don't allow
pipelining.

See the curl_multi_setopt() man page for details.

1 files changed, 111 insertions, 5 deletions

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 4915f2e81..cc6999cdc 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -140,6 +140,7 @@ my $GOPHER6PORT;         # Gopher IPv6 server port
 my $HTTPTLSPORT;         # HTTP TLS (non-stunnel) server port
 my $HTTPTLS6PORT;        # HTTP TLS (non-stunnel) IPv6 server port
 my $HTTPPROXYPORT;       # HTTP proxy port, when using CONNECT
+my $HTTPPIPEPORT;        # HTTP pipelining port
 
 my $srcdir = $ENV{'srcdir'} || '.';
 my $CURL="../src/curl".exe_ext(); # what curl executable to run on the tests
@@ -339,10 +340,10 @@ delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'});
 # Load serverpidfile hash with pidfile names for all possible servers.
 #
 sub init_serverpidfile_hash {
-  for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp')) {
+  for my $proto (('ftp', 'http', 'imap', 'pop3', 'smtp', 'http')) {
     for my $ssl (('', 's')) {
       for my $ipvnum ((4, 6)) {
-        for my $idnum ((1, 2)) {
+        for my $idnum ((1, 2, 3)) {
           my $serv = servername_id("$proto$ssl", $ipvnum, $idnum);
           my $pidf = server_pidfilename("$proto$ssl", $ipvnum, $idnum);
           $serverpidfile{$serv} = $pidf;
@@ -642,11 +643,11 @@ sub stopserver {
     # All servers relative to the given one must be stopped also
     #
     my @killservers;
-    if($server =~ /^(ftp|http|imap|pop3|smtp)s((\d*)(-ipv6|))$/) {
+    if($server =~ /^(ftp|http|imap|pop3|smtp|httppipe)s((\d*)(-ipv6|))$/) {
         # given a stunnel based ssl server, also kill non-ssl underlying one
         push @killservers, "${1}${2}";
     }
-    elsif($server =~ /^(ftp|http|imap|pop3|smtp)((\d*)(-ipv6|))$/) {
+    elsif($server =~ /^(ftp|http|imap|pop3|smtp|httppipe)((\d*)(-ipv6|))$/) {
         # given a non-ssl server, also kill stunnel based ssl piggybacking one
         push @killservers, "${1}s${2}";
     }
@@ -1105,6 +1106,7 @@ my %protofunc = ('http' => \&verifyhttp,
                  'pop3' => \&verifyftp,
                  'imap' => \&verifyftp,
                  'smtp' => \&verifyftp,
+                 'httppipe' => \&verifyhttp,
                  'ftps' => \&verifyftp,
                  'tftp' => \&verifyftp,
                  'ssh' => \&verifyssh,
@@ -1170,6 +1172,7 @@ sub runhttpserver {
     my $pidfile;
     my $logfile;
     my $flags = "";
+    my $exe = "$perl $srcdir/httpserver.pl";
 
     if($alt eq "ipv6") {
         # if IPv6, use a different setup
@@ -1180,6 +1183,11 @@ sub runhttpserver {
         # basically the same, but another ID
         $idnum = 2;
     }
+    elsif($alt eq "pipe") {
+        # basically the same, but another ID
+        $idnum = 3;
+	$exe = "python $srcdir/http_pipe.py";
+    }
 
     $server = servername_id($proto, $ipvnum, $idnum);
 
@@ -1207,7 +1215,82 @@ sub runhttpserver {
     $flags .= "--id $idnum " if($idnum > 1);
     $flags .= "--ipv$ipvnum --port $port --srcdir \"$srcdir\"";
 
-    my $cmd = "$perl $srcdir/httpserver.pl $flags";
+    my $cmd = "$exe $flags";
+    my ($httppid, $pid2) = startnew($cmd, $pidfile, 15, 0);
+
+    if($httppid <= 0 || !kill(0, $httppid)) {
+        # it is NOT alive
+        logmsg "RUN: failed to start the $srvrname server\n";
+        stopserver($server, "$pid2");
+        displaylogs($testnumcheck);
+        $doesntrun{$pidfile} = 1;
+        return (0,0);
+    }
+
+    # Server is up. Verify that we can speak to it.
+    my $pid3 = verifyserver($proto, $ipvnum, $idnum, $ip, $port);
+    if(!$pid3) {
+        logmsg "RUN: $srvrname server failed verification\n";
+        # failed to talk to it properly. Kill the server and return failure
+        stopserver($server, "$httppid $pid2");
+        displaylogs($testnumcheck);
+        $doesntrun{$pidfile} = 1;
+        return (0,0);
+    }
+    $pid2 = $pid3;
+
+    if($verbose) {
+        logmsg "RUN: $srvrname server is now running PID $httppid\n";
+    }
+
+    sleep(1);
+
+    return ($httppid, $pid2);
+}
+
+#######################################################################
+# start the http server
+#
+sub runhttp_pipeserver {
+    my ($proto, $verbose, $alt, $port) = @_;
+    my $ip = $HOSTIP;
+    my $ipvnum = 4;
+    my $idnum = 1;
+    my $server;
+    my $srvrname;
+    my $pidfile;
+    my $logfile;
+    my $flags = "";
+
+    if($alt eq "ipv6") {
+        # No IPv6
+    }
+
+    $server = servername_id($proto, $ipvnum, $idnum);
+
+    $pidfile = $serverpidfile{$server};
+
+    # don't retry if the server doesn't work
+    if ($doesntrun{$pidfile}) {
+        return (0,0);
+    }
+
+    my $pid = processexists($pidfile);
+    if($pid > 0) {
+        stopserver($server, "$pid");
+    }
+    unlink($pidfile) if(-f $pidfile);
+
+    $srvrname = servername_str($proto, $ipvnum, $idnum);
+
+    $logfile = server_logfilename($LOGDIR, $proto, $ipvnum, $idnum);
+
+    $flags .= "--verbose " if($debugprotocol);
+    $flags .= "--pidfile \"$pidfile\" --logfile \"$logfile\" ";
+    $flags .= "--id $idnum " if($idnum > 1);
+    $flags .= "--port $port --srcdir \"$srcdir\"";
+
+    my $cmd = "$srcdir/http_pipe.py $flags";
     my ($httppid, $pid2) = startnew($cmd, $pidfile, 15, 0);
 
     if($httppid <= 0 || !kill(0, $httppid)) {
@@ -2276,6 +2359,9 @@ sub checksystem {
             # 'http-proxy' is used in test cases to do CONNECT through
             push @protocols, 'http-proxy';
 
+            # 'http-pipe' is the special server for testing pipelining
+            push @protocols, 'http-pipe';
+
             # 'none' is used in test cases to mean no server
             push @protocols, 'none';
         }
@@ -2477,6 +2563,7 @@ sub checksystem {
         }
         logmsg "\n";
     }
+    logmsg sprintf("*   HTTP-PIPE/%d \n", $HTTPPIPEPORT);
 
     $has_textaware = ($^O eq 'MSWin32') || ($^O eq 'msys');
 
@@ -2505,6 +2592,7 @@ sub subVariables {
   $$thing =~ s/%HTTP6PORT/$HTTP6PORT/g;
   $$thing =~ s/%HTTPSPORT/$HTTPSPORT/g;
   $$thing =~ s/%HTTPPORT/$HTTPPORT/g;
+  $$thing =~ s/%HTTPPIPEPORT/$HTTPPIPEPORT/g;
   $$thing =~ s/%PROXYPORT/$HTTPPROXYPORT/g;
 
   $$thing =~ s/%IMAP6PORT/$IMAP6PORT/g;
@@ -3870,6 +3958,23 @@ sub startservers {
                 $run{'http-ipv6'}="$pid $pid2";
             }
         }
+        elsif($what eq "http-pipe") {
+            if($torture && $run{'http-pipe'} &&
+               !responsive_http_server("http", $verbose, "pipe",
+                                       $HTTPPIPEPORT)) {
+                stopserver('http-pipe');
+            }
+            if(!$run{'http-pipe'}) {
+                ($pid, $pid2) = runhttpserver("http", $verbose, "pipe",
+                                              $HTTPPIPEPORT);
+                if($pid <= 0) {
+                    return "failed starting HTTP-pipe server";
+                }
+                logmsg sprintf ("* pid http-pipe => %d %d\n", $pid, $pid2)
+                    if($verbose);
+                $run{'http-pipe'}="$pid $pid2";
+            }
+        }
         elsif($what eq "rtsp") {
             if($torture && $run{'rtsp'} &&
                !responsive_rtsp_server($verbose)) {
@@ -4512,6 +4617,7 @@ $GOPHER6PORT     = $base++; # Gopher IPv6 server port
 $HTTPTLSPORT     = $base++; # HTTP TLS (non-stunnel) server port
 $HTTPTLS6PORT    = $base++; # HTTP TLS (non-stunnel) IPv6 server port
 $HTTPPROXYPORT   = $base++; # HTTP proxy port, when using CONNECT
+$HTTPPIPEPORT    = $base++; # HTTP pipelining port
 
 #######################################################################
 # clear and create logging directory: