The hash of running servers is now a hash of hashes which for each running

server holds not only its two main pids, but also the pidfile of the test
server and the 'slavepidfiles' for ftp* servers. This allows a better control
when stopping servers.

Now from runtests.pl when test servers are stopped they are signalled in
sequence TERM, INT and KILL allowing time in between for them to die. This
will give us a chance of gracefully stopping test servers, which we didn't
have when we were killing them in first instance.

3 files changed, 526 insertions, 174 deletions

diff --git a/tests/ftp.pm b/tests/ftp.pm
index 015f9f08a..32c6779e3 100644
--- a/tests/ftp.pm
+++ b/tests/ftp.pm
@@ -1,63 +1,359 @@
-#######################################################################
-# Return the pid of the server as found in the given pid file
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
 #
-sub serverpid {
-    my $PIDFILE = $_[0];
-    open(PFILE, "<$PIDFILE");
-    my $PID=0+<PFILE>;
-    close(PFILE);
-    return $PID;
-}
+# Copyright (C) 1998 - 2006, Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at http://curl.haxx.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# $Id$
+###########################################################################
+
+use strict;
+use warnings;
 
-#######################################################################
-# Check the given test server if it is still alive.
+
+my $DEFAULT_TIMEOUT_START = 90; # default allowed time for a process to startup
+my $DEFAULT_TIMEOUT_STOP  = 90; # default allowed time for a process to stop
+
+my $ONE_HALF_STOP_TIMEOUT  = int($DEFAULT_TIMEOUT_STOP / 2);
+my $ONE_THIRD_STOP_TIMEOUT = int($DEFAULT_TIMEOUT_STOP / 3);
+my $ONE_SIXTH_STOP_TIMEOUT = int($DEFAULT_TIMEOUT_STOP / 6);
+
+my $pidpattern = qr/^\-?(\d+)$/; # pre-compiled pid pattern regexp
+
+
+######################################################################
+# pidfromfile returns the pid stored in the given pidfile.  The value
+# of the returned pid will never be a negative value. It will be zero
+# on any file related error or if a pid can not be extracted from the
+# file. Otherwise it will be a positive value, even If the pid number
+# stored in the file is a negative one.
 #
-sub checkserver {
+sub pidfromfile {
     my ($pidfile)=@_;
-    my $pid=0;
 
-    # check for pidfile
-    if ( -f $pidfile ) {
-        $pid=serverpid($pidfile);
-        if ($pid ne "" && kill(0, $pid)) {
-            return $pid;
+    my $pid = 0; # on failure or empty file return 0
+    my $pidline;
+
+    if(not defined $pidfile) {
+        return 0;
+    }
+    if(-f $pidfile) {
+        if(open(PIDF, "<$pidfile")) {
+            my $pidline = <PIDF>;
+            close(PIDF);
+            chomp $pidline;
+            $pidline =~ s/^\s+//;
+            $pidline =~ s/\s+$//;
+            $pidline =~ s/^[+-]?0+//;
+            if($pidline =~ $pidpattern) {
+                $pid = $1;
+            }
+        }
+    }
+    return $pid;
+}
+
+
+######################################################################
+# unlinkpidfiles unlinks/deletes the given pidfiles. The first argument
+# 'pidfiles' is a string of whitespace separated pidfiles. If successful
+# returns 0, on error it returns the number of files not deleted.
+#
+sub unlinkpidfiles {
+    my ($pidfiles)=@_;
+
+    if(not defined $pidfiles) {
+        return 0;
+    }
+    my $pidfile;
+    my $errcount = 0;
+    for $pidfile (split(" ", $pidfiles)) {
+        if($pidfile) {
+            if(unlink($pidfile) == 0) {
+                $errcount++;
+            }
+        }
+    }
+    return $errcount;
+}
+
+
+######################################################################
+# checkalivepid checks if the process of the given pid is alive. The
+# argument must represent a single pid and be a valid number, if not
+# it will return 0. It will also return 0 if the pid argument is zero
+# or negative. If the pid argument is positive and it is alive returns
+# the same positive pid, otherwise, if it is not alive it will return
+# the negative value of the pid argument.
+#
+sub checkalivepid {
+    my ($pid)=@_;
+
+    if(not defined $pid) {
+        return 0;
+    }
+    if ($pid !~ $pidpattern) {
+        return 0; # invalid argument
+    }
+    if($pid > 0) {
+        if(kill(0, $pid)) {
+            return $pid; # positive means it is alive
         }
         else {
             return -$pid; # negative means dead process
         }
     }
-    return 0;
+    return 0; # not a positive pid argument
+}
+
+
+######################################################################
+# checkalivepidfile checks if the process of the pid stored in the
+# given pidfile is alive. It will return 0 on any file related error
+# or if a pid can not be extracted from the file. If the process of
+# the pid present in the file is alive it returns that positive pid,
+# if it is not alive it will return the negative value of the pid.
+#
+sub checkalivepidfile {
+    my ($pidfile)=@_;
+
+    my $pid = pidfromfile($pidfile);
+    my $ret = checkalivepid($pid);
+    return $ret;
+}
+
+
+######################################################################
+# signalpids signals processes in the second argument with the signal
+# given in the first argument. The second argument 'pids' is a string
+# of whitespace separated pids. Of the given pids only those that are
+# positive and are actually alive will be signalled, and no matter
+# how many times a pid is repeated it will only be signalled once.
+#
+sub signalpids {
+    my ($signal, $pids)=@_;
+
+    if((not defined $signal) || (not defined $pids)) {
+        return;
+    }
+    my $prev = 0;
+    for(sort({$a <=> $b} split(" ", $pids))) {
+        if($_ =~ $pidpattern) {
+            my $pid = $1;
+            if($prev != $pid) {
+                $prev = $pid;
+                if(checkalivepid($pid) > 0) {
+                    kill($signal, $pid);
+                }
+            }
+        }
+    }
+}
+
+
+######################################################################
+# signalpidfile signals the process of the pid stored in the given
+# pidfile with the signal given in the first argument if the process
+# with that pid is actually alive.
+#
+sub signalpidfile {
+    my ($signal, $pidfile)=@_;
+
+    my $pid = pidfromfile($pidfile);
+    if($pid > 0) {
+        signalpids($signal, $pid);
+    }
+}
+
+    
+######################################################################
+# waitdeadpid waits until all processes given in the first argument
+# are not alive, waiting at most timeout seconds. The first argument
+# 'pids' is a string of whitespace separated pids. Returns 1 when all
+# pids are not alive. Returns 0 when the specified timeout has expired
+# and at least one of the specified pids is still alive.
+#
+sub waitdeadpid {
+    my ($pids, $timeout)=@_;
+
+    if(not defined $pids) {
+        return 1;
+    }
+    if((not defined $timeout) || ($timeout < 1)) {
+        $timeout = $DEFAULT_TIMEOUT_STOP;
+    }
+    while($timeout--) {
+        my $alive = 0;
+        for(split(" ", $pids)) {
+            if($_ =~ $pidpattern) {
+                my $pid = $1;
+                if(checkalivepid($pid) > 0) {
+                    $alive++;
+                }
+            }
+        }
+        if($alive == 0) {
+            return 1; # not a single pid is alive
+        }
+        sleep(1);
+    }
+    return 0; # at least one pid is still alive after timeout seconds
 }
 
-#############################################################################
-# Kill a specific slave
+
+######################################################################
+# waitalivepidfile waits until the given pidfile has a pid that is
+# alive, waiting at most timeout seconds. It returns the positive pid
+# When it is alive, otherwise it returns 0 when timeout seconds have
+# elapsed and the pidfile does not have a pid that is alive.
+#
+sub waitalivepidfile {
+    my ($pidfile, $timeout)=@_;
+
+    if(not defined $pidfile) {
+        return 0;
+    }
+    if((not defined $timeout) || ($timeout < 1)) {
+        $timeout = $DEFAULT_TIMEOUT_START;
+    }
+    while($timeout--) {
+        my $pid = checkalivepidfile($pidfile);
+        if($pid > 0) {
+            return $pid; # positive means it is alive
+        }
+        sleep(1);
+    }
+    return 0; # no pid in pidfile or not alive
+}
+
+
+######################################################################
+# stopprocess ends the given pid(s), waiting for them to die. The 'pids'
+# argument is a string of whitespace separated pids. Returns 1 if all
+# of the processes have been successfully stopped. If unable to stop
+# any of them in DEFAULT_TIMEOUT_STOP seconds then it returns 0.
+#
+sub stopprocess {
+    my ($pids)=@_;
+
+    if(not defined $pids) {
+        return 1;
+    }
+    signalpids("TERM", $pids);
+    if(waitdeadpid($pids, $ONE_HALF_STOP_TIMEOUT) == 0) {
+        signalpids("INT", $pids);
+        if(waitdeadpid($pids, $ONE_THIRD_STOP_TIMEOUT) == 0) {
+            signalpids("KILL", $pids);
+            if(waitdeadpid($pids, $ONE_SIXTH_STOP_TIMEOUT) == 0) {
+                return 0; # at least one pid is still alive !!!
+            }
+        }
+    }
+    return 1; # not a single pid is alive
+}
+
+
+######################################################################
+# stopprocesspidfile ends the test server process of the given pidfile,
+# waiting for it to die, and unlinking/deleting the given pidfile. If
+# the given process was not running or has been successfully stopped it
+# returns 1. If unable to stop it in DEFAULT_TIMEOUT_STOP seconds then
+# returns 0.
+#
+sub stopprocesspidfile {
+    my ($pidfile)=@_;
+
+    if(not defined $pidfile) {
+        return 1;
+    }
+    my $ret = 1; # assume success stopping it
+    my $pid = checkalivepidfile($pidfile);
+    if($pid > 0) {
+        $ret = stopprocess($pid);
+    }
+    unlinkpidfiles($pidfile);
+    return $ret;
+}
+
+
+######################################################################
+# ftpkillslave ends a specific slave, waiting for it to die, and
+# unlinking/deleting its pidfiles. If the given ftpslave was not
+# running or has been successfully stopped it returns 1. If unable
+# to stop it in DEFAULT_TIMEOUT_STOP seconds then it returns 0.
 #
 sub ftpkillslave {
-    my ($id, $ext, $verbose)=@_;
-    my $base;
-    for $base (('filt', 'data')) {
-        my $f = ".sock$base$id$ext.pid";
-        my $pid = checkserver($f);
+    my ($id, $ext)=@_;
+
+    my $ret = 1; # assume success stopping them
+    my $pids = "";
+    my $pidfiles = "";
+    for my $base (('filt', 'data')) {
+        my $pidfile = ".sock$base$id$ext.pid";
+        my $pid = checkalivepidfile($pidfile);
+        $pidfiles .= " $pidfile";
         if($pid > 0) {
-            printf ("* kill pid for %s => %d\n", "ftp-$base$id$ext", $pid) if($verbose);
-            kill (9, $pid); # die!
+            $pids .= " $pid";
         }
-        unlink($f);
     }
+    if($pids) {
+        $ret = stopprocess($pids);
+    }
+    if($pidfiles) {
+        unlinkpidfiles($pidfiles);
+    }
+    return $ret;
 }
 
 
-#############################################################################
-# Make sure no FTP leftovers are still running. Kill all slave processes.
-# This uses pidfiles since it might be used by other processes.
+######################################################################
+# ftpkillslaves ends all the ftpslave processes, waiting for them to
+# die, unlinking/deleting its pidfiles. If they were not running or
+# have been successfully stopped it returns 1. If unable to stop any
+# of them in DEFAULT_TIMEOUT_STOP seconds then returns 0.
 #
 sub ftpkillslaves {
-    my ($versbose) = @_;
-    for $ext (("", "ipv6")) {
-        for $id (("", "2")) {
-            ftpkillslave ($id, $ext, $verbose);
+
+    my $ret = 1; # assume success stopping them
+    my $pids = "";
+    my $pidfiles = "";
+    for my $ext (("", "ipv6")) {
+        for my $id (("", "2")) {
+            for my $base (('filt', 'data')) {
+                my $pidfile = ".sock$base$id$ext.pid";
+                my $pid = checkalivepidfile($pidfile);
+                $pidfiles .= " $pidfile";
+                if($pid > 0) {
+                    $pids .= " $pid";
+                }
+            }
         }
     }
+    if($pids) {
+        $ret = stopprocess($pids);
+    }
+    if($pidfiles) {
+        unlinkpidfiles($pidfiles);
+    }
+    return $ret;
 }
 
+
+######################################################################
+# library files end with 1; to make 'require' and 'use' succeed.
 1;
+
diff --git a/tests/ftpserver.pl b/tests/ftpserver.pl
index 0882dac37..fb64fff25 100644
--- a/tests/ftpserver.pl
+++ b/tests/ftpserver.pl
@@ -114,7 +114,7 @@ do {
 sub catch_zap {
     my $signame = shift;
     print STDERR "ftpserver.pl received SIG$signame, exiting\n";
-    ftpkillslaves(1);
+    ftpkillslaves();
     die "Somebody sent me a SIG$signame";
 }
 $SIG{INT} = \&catch_zap;
@@ -523,7 +523,7 @@ sub PASV_command {
     my $pasvport;
     my $pidf=".sockdata$ftpdnum$ext.pid";
 
-    my $prev = checkserver($pidf);
+    my $prev = checkalivepidfile($pidf);
     if($prev > 0) {
         print "kill existing server: $prev\n" if($verbose);
         kill(9, $prev);
diff --git a/tests/runtests.pl b/tests/runtests.pl
index d43657131..7b4196d54 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -73,6 +73,7 @@ my $TESTCASES="all";
 my $HTTPPIDFILE=".http.pid";
 my $HTTP6PIDFILE=".http6.pid";
 my $HTTPSPIDFILE=".https.pid";
+my $HTTPS6PIDFILE=".https6.pid";
 my $FTPPIDFILE=".ftp.pid";
 my $FTP6PIDFILE=".ftp6.pid";
 my $FTP2PIDFILE=".ftp2.pid";
@@ -197,7 +198,7 @@ $ENV{'HOME'}=$pwd;
 sub catch_zap {
     my $signame = shift;
     logmsg "runtests.pl received SIG$signame, exiting\n";
-    stopservers(1);
+    stopalltestservers();
     die "Somebody sent me a SIG$signame";
 }
 $SIG{INT} = \&catch_zap;
@@ -225,18 +226,30 @@ $ENV{'CURL_CA_BUNDLE'}=undef;
 
 #######################################################################
 # Start a new thread/process and run the given command line in there.
+# If successfully started an entry is added to the running servers hash.
 # Return the pids (yes plural) of the new child process to the parent.
 #
 sub startnew {
-    my ($cmd, $pidfile)=@_;
+    my ($cmd, $pidfile, $serv)=@_;
+
+    if((not defined $cmd) || (not defined $pidfile) || (not defined $serv)) {
+        return (-1,-1);
+    }
 
     logmsg "startnew: $cmd\n" if ($verbose);
 
+    my $UCSERV = uc($serv);
+
+    if(stoptestserver($serv) == 0) {
+        logmsg "RUN: failed to stop previous $UCSERV server!\n";
+        return (-1,-1);
+    }
+
     my $child = fork();
     my $pid2;
 
     if(not defined $child) {
-        logmsg "fork() failure detected\n";
+        logmsg "RUN: fork() failure detected for $UCSERV server!\n";
         return (-1,-1);
     }
 
@@ -252,25 +265,33 @@ sub startnew {
         # exec() should never return back here to this process. We protect
         # ourselfs calling die() just in case something goes really bad.
 
-        exec($cmd) || die "Can't exec() $cmd: $!";
+        exec($cmd) || die "Can't exec() cmd: $cmd";
 
-        die "error: exec() has returned !!!";
+        die "error: exec() has returned";
     }
 
-    my $count=5;
-    while($count--) {
-        if(-f $pidfile) {
-            open(PID, "<$pidfile");
-            $pid2 = 0 + <PID>;
-            close(PID);
-            if($pid2 && kill(0, $pid2)) {
-                # if $pid2 is valid, then make sure this pid is alive, as
-                # otherwise it is just likely to be the _previous_ pidfile or
-                # similar!
-                last;
-            }
-        }
-        sleep(1);
+    my $timeoutstart = 90; # seconds
+
+    $pid2 = waitalivepidfile($pidfile, $timeoutstart);
+    if(0 == $pid2) {
+        logmsg sprintf("RUN: server start timed out (%d sec) - ",
+                       $timeoutstart);
+        logmsg sprintf("%s server failed to start\n",
+                       $UCSERV);
+        return (-1,-1);
+    }
+
+    # setup entry in the running servers hash
+
+    $run{$serv}{'pidfile'} = $pidfile;    # pidfile for the test server.
+
+    $run{$serv}{'pids'} = "$child $pid2"; # forked pid and test server pid.
+
+    if($serv =~ /^ftp(\d*)(-ipv6|)/) {    # ftp servers have slavepidfiles.
+        my ($id, $ext) = ($1, $2);
+        $ext =~ s/\-//g;
+        my $slavepidfiles = ".sockfilt$id$ext.pid .sockdata$id$ext.pid";
+        $run{$serv}{'slavepidfiles'} = $slavepidfiles;
     }
 
     return ($child, $pid2);
@@ -385,7 +406,7 @@ sub torture {
         if($fail) {
             logmsg " Failed on alloc number $limit in test.\n",
             " invoke with -t$limit to repeat this single case.\n";
-            stopservers($verbose);
+            stopalltestservers();
             exit 1;
         }
     }
@@ -395,29 +416,61 @@ sub torture {
 }
 
 #######################################################################
-# stop the given test server (pid)
+# Stop specific test server processes, including slave processes, of the
+# given test server. Wait for them to finish and unlink its pidfiles.
+# If they were not running or have been successfully stopped return 1.
+# If unable to stop any of them then return 0. The test server is removed
+# from the running servers hash in any case.
 #
-sub stopserver {
-    my ($pid) = @_;
+sub stoptestserver {
+    my ($serv)=@_;
 
-    if(not defined $pid) {
-        return; # whad'da'ya wanna'da with no pid ?
+    if(not defined $serv) {
+        return 0;
     }
 
-    # it might be more than one pid
-
-    my @pids = split(/\s+/, $pid);
-    for (@pids) {
-        chomp($_);
-        if($_ =~ /^(\d+)$/) {
-            if(($1 > 0) && kill(0, $1)) {
-                if($verbose) {
-                    logmsg "RUN: Test server pid $1 signalled to die\n";
+    my $ret = 1; # assume success stopping them
+    my $pid;
+    my $pidfile;
+    my $pidfiles = "";
+    my $pidsrunning = "";
+
+    if($run{$serv}) {
+        if($run{$serv}{'slavepidfiles'}) {
+            for $pidfile (split(" ", $run{$serv}{'slavepidfiles'})) {
+                $pidfiles .= " $pidfile";
+                $pid = checkalivepidfile($pidfile);
+                if($pid > 0) {
+                    $pidsrunning .= " $pid";
                 }
-                kill(9, $1); # die!
             }
+            delete $run{$serv}{'slavepidfiles'};
+        }
+        if($run{$serv}{'pidfile'}) {
+            $pidfile = $run{$serv}{'pidfile'};
+            $pidfiles .= " $pidfile";
+            $pid = checkalivepidfile($pidfile);
+            if($pid > 0) {
+                $pidsrunning .= " $pid";
+            }
+            delete $run{$serv}{'pidfile'};
+        }
+        if($run{$serv}{'pids'}) {
+            $pid = $run{$serv}{'pids'};
+            $pidsrunning .= " $pid";
+            delete $run{$serv}{'pids'};
         }
+        if($run{$serv}) {
+            delete $run{$serv};
+        }
+    }
+    if($pidsrunning) {
+        $ret = stopprocess($pidsrunning);
+    }
+    if($pidfiles) {
+        unlinkpidfiles($pidfiles);
     }
+    return $ret;
 }
 
 #######################################################################
@@ -545,12 +598,12 @@ sub verifyserver {
 #
 sub runhttpserver {
     my ($verbose, $ipv6) = @_;
-    my $RUNNING;
-    my $pid;
+
     my $pidfile = $HTTPPIDFILE;
     my $port = $HTTPPORT;
     my $ip = $HOSTIP;
     my $nameext;
+    my $serv = "http";
     my $fork = $forkserver?"--fork":"";
 
     if($ipv6) {
@@ -559,13 +612,10 @@ sub runhttpserver {
         $port = $HTTP6PORT;
         $ip = $HOST6IP;
         $nameext="-ipv6";
+        $serv = "http-ipv6";
     }
 
-    $pid = checkserver($pidfile);
-
-    if($pid > 0) {
-        stopserver($pid);
-    }
+    my $UCSERV = uc($serv);
 
     my $flag=$debugprotocol?"-v ":"";
     my $dir=$ENV{'srcdir'};
@@ -574,26 +624,25 @@ sub runhttpserver {
     }
 
     my $cmd="$perl $srcdir/httpserver.pl -p $pidfile $fork$flag $port $ipv6";
-    my ($httppid, $pid2) =
-        startnew($cmd, $pidfile); # start the server in a new process
 
-    if(!kill(0, $httppid)) {
+    my ($httppid, $pid2) = startnew($cmd, $pidfile, $serv);
+
+    if((checkalivepid($httppid) <= 0) || (checkalivepid($pid2) <= 0)) {
         # it is NOT alive
-        logmsg "RUN: failed to start the HTTP server!\n";
-        stopservers($verbose);
+        logmsg "RUN: failed to start the $UCSERV server!\n";
+        stopalltestservers();
         return (0,0);
     }
 
     # Server is up. Verify that we can speak to it.
     if(!verifyserver("http", $ip, $port)) {
-        logmsg "RUN: HTTP$nameext server failed verification\n";
-        # failed to talk to it properly. Kill the server and return failure
-        stopserver("$httppid $pid2");
+        logmsg "RUN: $UCSERV server failed verification\n";
+        stopalltestservers();
         return (0,0);
     }
 
     if($verbose) {
-        logmsg "RUN: HTTP$nameext server is now running PID $httppid\n";
+        logmsg "RUN: $UCSERV server is now running PID $httppid\n";
     }
 
     sleep(1);
@@ -606,9 +655,10 @@ sub runhttpserver {
 #
 sub runhttpsserver {
     my ($verbose, $ipv6) = @_;
-    my $STATUS;
-    my $RUNNING;
+
+    my $pidfile = $HTTPSPIDFILE;
     my $ip = $HOSTIP;
+    my $serv = "https";
 
     if(!$stunnel) {
         return 0;
@@ -616,38 +666,34 @@ sub runhttpsserver {
 
     if($ipv6) {
         # not complete yet
+        $pidfile = $HTTPS6PIDFILE;
         $ip = $HOST6IP;
+        $serv = "https-ipv6";
     }
 
-    my $pid=checkserver($HTTPSPIDFILE);
-
-    if($pid > 0) {
-        # kill previous stunnel!
-        stopserver($pid);
-    }
+    my $UCSERV = uc($serv);
 
     my $flag=$debugprotocol?"-v ":"";
     my $cmd="$perl $srcdir/httpsserver.pl $flag -s \"$stunnel\" -d $srcdir -r $HTTPPORT $HTTPSPORT";
 
-    my ($httpspid, $pid2) = startnew($cmd, $HTTPSPIDFILE);
+    my ($httpspid, $pid2) = startnew($cmd, $pidfile, $serv);
 
-    if(!kill(0, $httpspid)) {
+    if((checkalivepid($httpspid) <= 0) || (checkalivepid($pid2) <= 0)) {
         # it is NOT alive
-        logmsg "RUN: failed to start the HTTPS server!\n";
-        stopservers($verbose);
-        return(0,0);
+        logmsg "RUN: failed to start the $UCSERV server!\n";
+        stopalltestservers();
+        return (0,0);
     }
 
     # Server is up. Verify that we can speak to it.
     if(!verifyserver("https", $ip, $HTTPSPORT)) {
-        logmsg "RUN: HTTPS server failed verification\n";
-        # failed to talk to it properly. Kill the server and return failure
-        stopserver("$httpspid $pid2");
+        logmsg "RUN: $UCSERV server failed verification\n";
+        stopalltestservers();
         return (0,0);
     }
 
     if($verbose) {
-        logmsg "RUN: HTTPS server is now running PID $httpspid\n";
+        logmsg "RUN: $UCSERV server is now running PID $httpspid\n";
     }
 
     sleep(1);
@@ -660,13 +706,13 @@ sub runhttpsserver {
 #
 sub runftpserver {
     my ($id, $verbose, $ipv6) = @_;
-    my $STATUS;
-    my $RUNNING;
+
     my $port = $id?$FTP2PORT:$FTPPORT;
     # check for pidfile
     my $pidfile = $id?$FTP2PIDFILE:$FTPPIDFILE;
     my $ip=$HOSTIP;
     my $nameext;
+    my $serv = "ftp$id";
     my $cmd;
 
     if($ipv6) {
@@ -675,12 +721,10 @@ sub runftpserver {
         $port = $FTP6PORT;
         $ip = $HOST6IP;
         $nameext="-ipv6";
+        $serv = "ftp$id-ipv6";
     }
 
-    my $pid = checkserver($pidfile);
-    if($pid >= 0) {
-        stopserver($pid);
-    }
+    my $UCSERV = uc($serv);
 
     # start our server:
     my $flag=$debugprotocol?"-v ":"";
@@ -693,26 +737,24 @@ sub runftpserver {
     }
     $cmd="$perl $srcdir/ftpserver.pl --pidfile $pidfile $flag --port $port";
 
-    unlink($pidfile);
+    my ($ftppid, $pid2) = startnew($cmd, $pidfile, $serv);
 
-    my ($ftppid, $pid2) = startnew($cmd, $pidfile);
-
-    if(!$ftppid || !kill(0, $ftppid)) {
+    if((checkalivepid($ftppid) <= 0) || (checkalivepid($pid2) <= 0)) {
         # it is NOT alive
-        logmsg "RUN: failed to start the FTP$id$nameext server!\n";
-        return -1;
+        logmsg "RUN: failed to start the $UCSERV server!\n";
+        stopalltestservers();
+        return (0,0);
     }
 
     # Server is up. Verify that we can speak to it.
     if(!verifyserver("ftp", $ip, $port)) {
-        logmsg "RUN: FTP$id$nameext server failed verification\n";
-        # failed to talk to it properly. Kill the server and return failure
-        stopserver("$ftppid $pid2");
+        logmsg "RUN: $UCSERV server failed verification\n";
+        stopalltestservers();
         return (0,0);
     }
 
     if($verbose) {
-        logmsg "RUN: FTP$id$nameext server is now running PID $ftppid\n";
+        logmsg "RUN: $UCSERV server is now running PID $ftppid\n";
     }
 
     sleep(1);
@@ -725,13 +767,13 @@ sub runftpserver {
 #
 sub runtftpserver {
     my ($id, $verbose, $ipv6) = @_;
-    my $STATUS;
-    my $RUNNING;
+
     my $port = $TFTPPORT;
     # check for pidfile
     my $pidfile = $TFTPPIDFILE;
     my $ip=$HOSTIP;
     my $nameext;
+    my $serv = "tftp$id";
     my $cmd;
 
     if($ipv6) {
@@ -740,12 +782,10 @@ sub runtftpserver {
         $port = $TFTP6PORT;
         $ip = $HOST6IP;
         $nameext="-ipv6";
+        $serv = "tftp$id-ipv6";
     }
 
-    my $pid = checkserver($pidfile);
-    if($pid >= 0) {
-        stopserver($pid);
-    }
+    my $UCSERV = uc($serv);
 
     # start our server:
     my $flag=$debugprotocol?"-v ":"";
@@ -758,26 +798,24 @@ sub runtftpserver {
     }
     $cmd="./server/tftpd --pidfile $pidfile $flag $port";
 
-    unlink($pidfile);
-
-    my ($tftppid, $pid2) = startnew($cmd, $pidfile);
+    my ($tftppid, $pid2) = startnew($cmd, $pidfile, $serv);
 
-    if(!$tftppid || !kill(0, $tftppid)) {
+    if((checkalivepid($tftppid) <= 0) || (checkalivepid($pid2) <= 0)) {
         # it is NOT alive
-        logmsg "RUN: failed to start the FTP$id$nameext server!\n";
-        return -1;
+        logmsg "RUN: failed to start the $UCSERV server!\n";
+        stopalltestservers();
+        return (0,0);
     }
 
     # Server is up. Verify that we can speak to it.
     if(!verifyserver("tftp", $ip, $port)) {
-        logmsg "RUN: TFTP$id$nameext server failed verification\n";
-        # failed to talk to it properly. Kill the server and return failure
-        stopserver("$tftppid $pid2");
+        logmsg "RUN: $UCSERV server failed verification\n";
+        stopalltestservers();
         return (0,0);
     }
 
     if($verbose) {
-        logmsg "RUN: TFTP$id$nameext server is now running PID $tftppid\n";
+        logmsg "RUN: $UCSERV server is now running PID $tftppid\n";
     }
 
     sleep(1);
@@ -1437,7 +1475,7 @@ sub singletest {
         if($@) {
             logmsg "perl: $code\n";
             logmsg "precommand: $@";
-            stopservers($verbose);
+            stopalltestservers();
             return -1;
         }
     }
@@ -1628,7 +1666,7 @@ sub singletest {
         my $filename=$hash{'name'};
         if(!$filename) {
             logmsg "ERROR: section verify=>file has no name attribute!\n";
-            stopservers($verbose);
+            stopalltestservers();
             return -1;
         }
         my @generated=loadarray($filename);
@@ -1688,14 +1726,9 @@ sub singletest {
     for(@what) {
         my $serv = $_;
         chomp $serv;
-        if($serv =~ /^ftp(\d*)(-ipv6|)/) {
-            my ($id, $ext) = ($1, $2);
-            #print STDERR "SERV $serv $id $ext\n";
-            ftpkillslave($id, $ext, $verbose);
-        }
         if($run{$serv}) {
-            stopserver($run{$serv}); # the pid file is in the hash table
-            $run{$serv}=0; # clear pid
+            logmsg "RUN: Stopping the $serv server\n";
+            stoptestserver($serv);
         }
         else {
             logmsg "RUN: The $serv server is not running\n";
@@ -1792,26 +1825,58 @@ sub singletest {
 }
 
 #######################################################################
-# Stop all running test servers
-sub stopservers {
-    my ($verbose)=@_;
-    for(keys %run) {
-        my $server = $_;
-        my $pids=$run{$server};
-        my $pid;
-        my $prev;
-
-        foreach $pid (split(" ", $pids)) {
-            if($pid != $prev) {
-                # no need to kill same pid twice!
-                logmsg sprintf("* kill pid for %s => %d\n",
-                               $server, $pid) if($verbose);
-                stopserver($pid);
+# Stop all processes, including slave processes, of all the running test
+# servers. Wait for them to finish and unlink its pidfiles. If they were
+# not running or have been successfully stopped return 1. If unable to
+# stop any of them then return 0. In any case the running servers hash
+# is completely cleared for all test servers.
+#
+sub stopalltestservers {
+
+    my $ret = 1; # assume success stopping them
+    my $pid;
+    my $pidfile;
+    my $pidfiles = "";
+    my $pidsrunning = "";
+
+    for my $serv (keys %run) {
+        if($run{$serv}) {
+            if($run{$serv}{'slavepidfiles'}) {
+                for $pidfile (split(" ", $run{$serv}{'slavepidfiles'})) {
+                    $pidfiles .= " $pidfile";
+                    $pid = checkalivepidfile($pidfile);
+                    if($pid > 0) {
+                        $pidsrunning .= " $pid";
+                    }
+                }
+                delete $run{$serv}{'slavepidfiles'};
+            }
+            if($run{$serv}{'pidfile'}) {
+                $pidfile = $run{$serv}{'pidfile'};
+                $pidfiles .= " $pidfile";
+                $pid = checkalivepidfile($pidfile);
+                if($pid > 0) {
+                    $pidsrunning .= " $pid";
+                }
+                delete $run{$serv}{'pidfile'};
+            }
+            if($run{$serv}{'pids'}) {
+                $pid = $run{$serv}{'pids'};
+                $pidsrunning .= " $pid";
+                delete $run{$serv}{'pids'};
+            }
+            if($run{$serv}) {
+                delete $run{$serv};
             }
-            $prev = $pid;
         }
     }
-    ftpkillslaves($verbose);
+    if($pidsrunning) {
+        $ret = stopprocess($pidsrunning);
+    }
+    if($pidfiles) {
+        unlinkpidfiles($pidfiles);
+    }
+    return $ret;
 }
 
 #######################################################################
@@ -1832,7 +1897,6 @@ sub startservers {
                     return "failed starting FTP server";
                 }
                 printf ("* pid ftp => %d %d\n", $pid, $pid2) if($verbose);
-                $run{'ftp'}="$pid $pid2";
             }
         }
         elsif($what eq "ftp2") {
@@ -1842,7 +1906,6 @@ sub startservers {
                     return "failed starting FTP2 server";
                 }
                 printf ("* pid ftp2 => %d %d\n", $pid, $pid2) if($verbose);
-                $run{'ftp2'}="$pid $pid2";
             }
         }
         elsif($what eq "ftp-ipv6") {
@@ -1853,7 +1916,6 @@ sub startservers {
                 }
                 logmsg sprintf("* pid ftp-ipv6 => %d %d\n", $pid,
                        $pid2) if($verbose);
-                $run{'ftp-ipv6'}="$pid $pid2";
             }
         }
         elsif($what eq "http") {
@@ -1863,7 +1925,6 @@ sub startservers {
                     return "failed starting HTTP server";
                 }
                 printf ("* pid http => %d %d\n", $pid, $pid2) if($verbose);
-                $run{'http'}="$pid $pid2";
             }
         }
         elsif($what eq "http-ipv6") {
@@ -1874,7 +1935,6 @@ sub startservers {
                 }
                 logmsg sprintf("* pid http-ipv6 => %d %d\n", $pid, $pid2)
                     if($verbose);
-                $run{'http-ipv6'}="$pid $pid2";
             }
         }
         elsif($what eq "ftps") {
@@ -1900,7 +1960,6 @@ sub startservers {
                     return "failed starting HTTP server";
                 }
                 printf ("* pid http => %d %d\n", $pid, $pid2) if($verbose);
-                $run{'http'}="$pid $pid2";
             }
             if(!$run{'https'}) {
                 ($pid, $pid2) = runhttpsserver($verbose);
@@ -1909,7 +1968,6 @@ sub startservers {
                 }
                 logmsg sprintf("* pid https => %d %d\n", $pid, $pid2)
                     if($verbose);
-                $run{'https'}="$pid $pid2";
             }
         }
         elsif($what eq "tftp") {
@@ -1919,7 +1977,6 @@ sub startservers {
                     return "failed starting TFTP server";
                 }
                 printf ("* pid tftp => %d %d\n", $pid, $pid2) if($verbose);
-                $run{'tftp'}="$pid $pid2";
             }
         }
         elsif($what eq "tftp-ipv6") {
@@ -1929,7 +1986,6 @@ sub startservers {
                     return "failed starting TFTP-IPv6 server";
                 }
                 printf("* pid tftp-ipv6 => %d %d\n", $pid, $pid2) if($verbose);
-                $run{'tftp-ipv6'}="$pid $pid2";
             }
         }
         elsif($what eq "none") {
@@ -2263,7 +2319,7 @@ foreach $testnum (@at) {
 close(CMDLOG);
 
 # Tests done, stop the servers
-stopservers($verbose);
+stopalltestservers();
 
 my $all = $total + $skipped;