aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDaniel Stenberg <daniel@haxx.se>2019-12-10 14:29:48 +0100
committerDaniel Stenberg <daniel@haxx.se>2019-12-11 09:55:15 +0100
commit7c1bd035760faa5ad6d81b6280dffbdd7feb6302 (patch)
tree27f2ac04d28f4133bb033afb3f206f5ea67573fb
parent1d5c427d7fd6c206de4b010f618659a7a9b38cff (diff)
runtests: introduce --shallow to reduce huge torture tests
When set, shallow mode limits runtests -t to make no more than NUM fails per test case. If more are found, it will randomly discard entries until the number is right. The random seed can also be set. This is particularly useful when running MANY tests as then most torture failures will already fail the same functions over and over and make the total operation painfully tedious. Closes #4699
-rw-r--r--tests/runtests.17
-rwxr-xr-xtests/runtests.pl38
2 files changed, 41 insertions, 4 deletions
diff --git a/tests/runtests.1 b/tests/runtests.1
index 18ddce483..bb36a8b44 100644
--- a/tests/runtests.1
+++ b/tests/runtests.1
@@ -5,7 +5,7 @@
.\" * | (__| |_| | _ <| |___
.\" * \___|\___/|_| \_\_____|
.\" *
-.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
.\" *
.\" * This software is licensed as described in the file COPYING, which
.\" * you should have received as part of this distribution. The terms
@@ -84,6 +84,11 @@ Display run time statistics. (Requires Perl Time::HiRes module)
Display full run time statistics. (Requires Perl Time::HiRes module)
.IP "-s"
Shorter output. Speaks less than default.
+.IP "--shallow=[num](,seed)"
+Used together with \fB-t\fP. This limits the number of tests to fail in
+torture mode to no more than 'num' per test case. If this reduces the amount,
+the given 'seed' will be used to randomly discard entries to fail until the
+amount is 'num'.
.IP "-t[num]"
Selects a \fBtorture\fP test for the given tests. This makes runtests.pl first
run the tests once and count the number of memory allocations made. It then
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 961cdb753..6ee2cca20 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -321,6 +321,8 @@ my %runcert; # cert file currently in use by an ssl running server
my $torture;
my $tortnum;
my $tortalloc;
+my $shallow;
+my $shallowseed;
#######################################################################
# logmsg is our general message logging subroutine.
@@ -598,13 +600,34 @@ sub torture {
return 0;
}
- logmsg " $count functions to make fail\n";
+ my @ttests = (1 .. $count);
+ if($shallow && ($shallow < $count)) {
+ my $discard = scalar(@ttests) - $shallow;
+ my $percent = sprintf("%.2f%%", $shallow * 100 / scalar(@ttests));;
+ logmsg " $count functions found, but only fail $shallow ($percent)\n";
+ while($discard) {
+ my $rm;
+ do {
+ # find a test to discard
+ $rm = rand(scalar(@ttests));
+ } while(!$ttests[$rm]);
+ $ttests[$rm] = undef;
+ $discard--;
+ }
+ }
+ else {
+ logmsg " $count functions to make fail\n";
+ }
- for ( 1 .. $count ) {
+ for (@ttests) {
my $limit = $_;
my $fail;
my $dumped_core;
+ if(!defined($limit)) {
+ # --shallow can undefine them
+ next;
+ }
if($tortalloc && ($tortalloc != $limit)) {
next;
}
@@ -5022,6 +5045,14 @@ while(@ARGV) {
$tortalloc = $1;
}
}
+ elsif($ARGV[0] =~ /--shallow=(\d+)(,|)(\d*)/) {
+ # Fail no more than this amount per tests when running
+ # torture.
+ my ($num, $seed)=($1,$3);
+ $shallow=$num;
+ $shallowseed=$seed?$seed:1234; # get a real seed later
+ srand($shallowseed); # make it predictable
+ }
elsif($ARGV[0] eq "-a") {
# continue anyway, even if a test fail
$anyway=1;
@@ -5070,6 +5101,7 @@ while(@ARGV) {
print <<EOHELP
Usage: runtests.pl [options] [test selection(s)]
-a continue even if a test fails
+ -am automake style output PASS/FAIL: [number] [name]
-bN use base port number N for test servers (default $base)
-c path use this curl executable
-d display server debug info
@@ -5085,7 +5117,7 @@ Usage: runtests.pl [options] [test selection(s)]
-r run time statistics
-rf full run time statistics
-s short output
- -am automake style output PASS/FAIL: [number] [name]
+ --shallow=[num](,seed) make the torture tests thinner
-t[N] torture (simulate function failures); N means fail Nth function
-v verbose output
-vc path use this curl only to verify the existing servers