aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorMarc Hoersken <info@marc-hoersken.de>2020-03-02 14:05:59 +0100
committerMarc Hoersken <info@marc-hoersken.de>2020-03-03 17:32:38 +0100
commit119ea453f9c9854feac09d86039334d6aeb40fcf (patch)
treef27e09cfcc4698199c248d7a491932286318df30 /tests
parent86ceb9b006457d6a261f1aed8197f57086297f47 (diff)
ci/tests: Make it possible to still run but ignore failing tests
This enables the development of a solution for the failing tests by running them on CI while ignoring their result for the overall status. Closes #4994
Diffstat (limited to 'tests')
-rw-r--r--tests/appveyor.pm6
-rw-r--r--tests/azure.pm5
-rwxr-xr-xtests/runtests.pl58
3 files changed, 52 insertions, 17 deletions
diff --git a/tests/appveyor.pm b/tests/appveyor.pm
index abfb18523..6ed83dd22 100644
--- a/tests/appveyor.pm
+++ b/tests/appveyor.pm
@@ -63,7 +63,11 @@ sub appveyor_update_test_result {
my $appveyor_duration = sprintf("%.0f", ($stop-$start)*1000);
my $appveyor_outcome;
my $appveyor_category;
- if($error < 0) {
+ if($error == 2) {
+ $appveyor_outcome = 'Ignored';
+ $appveyor_category = 'Warning';
+ }
+ elsif($error < 0) {
$appveyor_outcome = 'NotRunnable';
$appveyor_category = 'Warning';
}
diff --git a/tests/azure.pm b/tests/azure.pm
index 2fc11ae21..94fe995d9 100644
--- a/tests/azure.pm
+++ b/tests/azure.pm
@@ -88,7 +88,10 @@ sub azure_update_test_result {
my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop;
my $azure_duration = sprintf("%.0f", ($stop-$start)*1000);
my $azure_outcome;
- if($error < 0) {
+ if($error == 2) {
+ $azure_outcome = 'Not applicable';
+ }
+ elsif($error < 0) {
$azure_outcome = 'Not executed';
}
elsif(!$error) {
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 51a84f09b..3306de397 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -276,8 +276,10 @@ my $skipped=0; # number of tests skipped; reported in main loop
my %skipped; # skipped{reason}=counter, reasons for skip
my @teststat; # teststat[testnum]=reason, reasons for skip
my %disabled_keywords; # key words of tests to skip
+my %ignored_keywords; # key words of tests to ignore results
my %enabled_keywords; # key words of tests to run
my %disabled; # disabled test cases
+my %ignored; # ignored results of test cases
my $sshdid; # for socks server, ssh daemon version id
my $sshdvernum; # for socks server, ssh daemon version number
@@ -3250,6 +3252,7 @@ sub singletest {
my $why;
my $cmd;
my $disablevalgrind;
+ my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
# fist, remove all lingering log files
cleardir($LOGDIR);
@@ -3267,6 +3270,10 @@ sub singletest {
if($disabled{$testnum}) {
logmsg "Warning: test$testnum is explicitly disabled\n";
}
+ if($ignored{$testnum}) {
+ logmsg "Warning: test$testnum result is ignored\n";
+ $errorreturncode = 2;
+ }
# load the test case file definition
if(loadtest("${TESTDIR}/test${testnum}")) {
@@ -3334,6 +3341,10 @@ sub singletest {
} elsif ($enabled_keywords{lc($k)}) {
$match = 1;
}
+ if ($ignored_keywords{lc($k)}) {
+ logmsg "Warning: test$testnum result is ignored due to $k\n";
+ $errorreturncode = 2;
+ }
$keywords{$k} = 1;
}
@@ -3918,7 +3929,7 @@ sub singletest {
logmsg " postcheck FAILED\n";
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time();
- return 1;
+ return $errorreturncode;
}
}
}
@@ -3990,7 +4001,7 @@ sub singletest {
$res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
if($res) {
- return 1;
+ return $errorreturncode;
}
$ok .= "s";
}
@@ -4041,7 +4052,7 @@ sub singletest {
$res = compare($testnum, $testname, "stderr", \@actual, \@validstderr);
if($res) {
- return 1;
+ return $errorreturncode;
}
$ok .= "r";
}
@@ -4087,7 +4098,7 @@ sub singletest {
$res = compare($testnum, $testname, "protocol", \@out, \@protstrip);
if($res) {
- return 1;
+ return $errorreturncode;
}
$ok .= "p";
@@ -4102,7 +4113,7 @@ sub singletest {
my @out = loadarray($CURLOUT);
$res = compare($testnum, $testname, "data", \@out, \@reply);
if ($res) {
- return 1;
+ return $errorreturncode;
}
$ok .= "d";
}
@@ -4126,7 +4137,7 @@ sub singletest {
$res = compare($testnum, $testname, "upload", \@out, \@upload);
if ($res) {
- return 1;
+ return $errorreturncode;
}
$ok .= "u";
}
@@ -4172,7 +4183,7 @@ sub singletest {
$res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
if($res) {
- return 1;
+ return $errorreturncode;
}
$ok .= "P";
@@ -4230,7 +4241,7 @@ sub singletest {
$res = compare($testnum, $testname, "output ($filename)",
\@generated, \@outfile);
if($res) {
- return 1;
+ return $errorreturncode;
}
$outputok = 1; # output checked
@@ -4260,7 +4271,7 @@ sub singletest {
logmsg " exit FAILED\n";
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time();
- return 1;
+ return $errorreturncode;
}
if($has_memory_tracking) {
@@ -4283,7 +4294,7 @@ sub singletest {
logmsg @memdata;
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time();
- return 1;
+ return $errorreturncode;
}
else {
$ok .= "m";
@@ -4300,7 +4311,7 @@ sub singletest {
logmsg "ERROR: unable to read $LOGDIR\n";
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time();
- return 1;
+ return $errorreturncode;
}
my @files = readdir(DIR);
closedir(DIR);
@@ -4315,7 +4326,7 @@ sub singletest {
logmsg "ERROR: valgrind log file missing for test $testnum\n";
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time();
- return 1;
+ return $errorreturncode;
}
my @e = valgrindparse("$LOGDIR/$vgfile");
if(@e && $e[0]) {
@@ -4328,7 +4339,7 @@ sub singletest {
}
# timestamp test result verification end
$timevrfyend{$testnum} = Time::HiRes::time();
- return 1;
+ return $errorreturncode;
}
$ok .= "v";
}
@@ -4367,6 +4378,10 @@ sub singletest {
logmsg "PASS: $testnum - $testname\n";
}
+ if($errorreturncode==2) {
+ logmsg "Warning: test$testnum result is ignored, but passed!\n";
+ }
+
return 0;
}
@@ -5164,8 +5179,10 @@ Usage: runtests.pl [options] [test selection(s)]
-vc path use this curl only to verify the existing servers
[num] like "5 6 9" or " 5 to 22 " to run those tests only
[!num] like "!5 !6 !9" to disable those tests
+ [~num] like "~5 ~6 ~9" to ignore the result of those tests
[keyword] like "IPv6" to select only tests containing the key word
[!keyword] like "!cookies" to disable any tests containing the key word
+ [~keyword] like "~cookies" to ignore results of tests containing key word
EOHELP
;
exit;
@@ -5198,9 +5215,16 @@ EOHELP
$fromnum = -1;
$disabled{$1}=$1;
}
+ elsif($ARGV[0] =~ /^~(\d+)/) {
+ $fromnum = -1;
+ $ignored{$1}=$1;
+ }
elsif($ARGV[0] =~ /^!(.+)/) {
$disabled_keywords{lc($1)}=$1;
}
+ elsif($ARGV[0] =~ /^~(.+)/) {
+ $ignored_keywords{lc($1)}=$1;
+ }
elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
$enabled_keywords{lc($1)}=$1;
}
@@ -5532,6 +5556,7 @@ if(azure_check_environment()) {
my $failed;
my $testnum;
my $ok=0;
+my $ign=0;
my $total=0;
my $lasttest=0;
my @at = split(" ", $TESTCASES);
@@ -5568,7 +5593,10 @@ foreach $testnum (@at) {
# display all files in log/ in a nice way
displaylogs($testnum);
}
- if(!$anyway) {
+ if($error==2) {
+ $ign++; # ignored test result counter
+ }
+ elsif(!$anyway) {
# a test failed, abort
logmsg "\n - abort tests\n";
last;
@@ -5644,6 +5672,6 @@ if($skipped && !$short) {
}
}
-if($total && ($ok != $total)) {
+if($total && (($ok+$ign) != $total)) {
exit 1;
}