aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--tests/runtests.14
-rwxr-xr-xtests/runtests.pl29
2 files changed, 27 insertions, 6 deletions
diff --git a/tests/runtests.1 b/tests/runtests.1
index 5576b2fd8..02540c84b 100644
--- a/tests/runtests.1
+++ b/tests/runtests.1
@@ -5,7 +5,7 @@
.\" * | (__| |_| | _ <| |___
.\" * \___|\___/|_| \_\_____|
.\" *
-.\" * Copyright (C) 1998 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
.\" *
.\" * This software is licensed as described in the file COPYING, which
.\" * you should have received as part of this distribution. The terms
@@ -71,6 +71,8 @@ detected. Useful for debugging.
Lists all test case names.
.IP "-n"
Disable the check for and use of valgrind.
+.IP "-o"
+Run the tests in a scrambled, or randomized, order instead of sequentially.
.IP "-p"
Prints out all files in "log/" to stdout when a test case fails. Very
practical when used in the automated and distributed tests since then the
diff --git a/tests/runtests.pl b/tests/runtests.pl
index e281e318c..d3e4fbbcc 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -194,7 +194,7 @@ my $pwd = getcwd(); # current working directory
my $start;
my $ftpchecktime=1; # time it took to verify our test FTP server
-
+my $scrambleorder;
my $stunnel = checkcmd("stunnel4") || checkcmd("tstunnel") || checkcmd("stunnel");
my $valgrind = checktestcmd("valgrind");
my $valgrind_logfile="--logfile";
@@ -4870,6 +4870,10 @@ while(@ARGV) {
# have the servers display protocol output
$debugprotocol=1;
}
+ elsif($ARGV[0] eq "-e") {
+ # run the tests cases event based if possible
+ $run_event_based=1;
+ }
elsif ($ARGV[0] eq "-g") {
# run this test with gdb
$gdbthis=1;
@@ -4892,6 +4896,10 @@ while(@ARGV) {
# no valgrind
undef $valgrind;
}
+ elsif ($ARGV[0] eq "-o") {
+ # execute in scrambled order
+ $scrambleorder=1;
+ }
elsif($ARGV[0] =~ /^-t(.*)/) {
# torture
$torture=1;
@@ -4907,10 +4915,6 @@ while(@ARGV) {
# continue anyway, even if a test fail
$anyway=1;
}
- elsif($ARGV[0] eq "-e") {
- # run the tests cases event based if possible
- $run_event_based=1;
- }
elsif($ARGV[0] eq "-p") {
$postmortem=1;
}
@@ -4958,12 +4962,14 @@ Usage: runtests.pl [options] [test selection(s)]
-bN use base port number N for test servers (default $base)
-c path use this curl executable
-d display server debug info
+ -e event-based execution
-g run the test case with gdb
-gw run the test case with gdb as a windowed application
-h this help text
-k keep stdout and stderr files present after tests
-l list all test case names/descriptions
-n no valgrind
+ -o scrambled order
-p print log file contents when a test fails
-r run time statistics
-rf full run time statistics
@@ -5188,6 +5194,19 @@ else {
$TESTCASES = $verified;
}
+if($scrambleorder) {
+ # scramble the order of the test cases
+ my @rand;
+ while($TESTCASES) {
+ my @all = split(/ /, $TESTCASES);
+ my $r = rand @all;
+ push @rand, $all[$r];
+ $all[$r]="";
+ $TESTCASES = join(" ", @all);
+ }
+ $TESTCASES = join(" ", @rand);
+}
+
#######################################################################
# Start the command line log
#