diff options
| -rw-r--r-- | .azure-pipelines.yml | 34 | ||||
| -rw-r--r-- | tests/azure.pm | 140 | ||||
| -rwxr-xr-x | tests/runtests.pl | 33 |
3 files changed, 206 insertions, 1 deletions
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 054df62b3..e8ab8f397 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -29,6 +29,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: disable_ipv6 displayName: ubuntu w/o IPv6 @@ -43,6 +45,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: disable_http_smtp_imap displayName: ubuntu w/o HTTP/SMTP/IMAP @@ -57,6 +61,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: disable_thredres displayName: ubuntu sync resolver @@ -71,6 +77,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: http_only displayName: ubuntu HTTP only @@ -85,6 +93,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - stage: linux_torture dependsOn: linux @@ -129,6 +139,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: macos_libssh2 displayName: macos libssh2 @@ -146,6 +158,8 @@ stages: - script: make test-nonflaky displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: macos_cmake displayName: macos cmake openssl @@ -205,6 +219,8 @@ stages: - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys2_mingw64_debug_openssl displayName: msys2 mingw64 debug openssl @@ -226,6 +242,8 @@ stages: - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys1_mingw_debug_openssl displayName: msys1 mingw debug openssl @@ -245,6 +263,8 @@ stages: - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys1_mingw32_debug_openssl displayName: msys1 mingw32 debug openssl @@ -264,6 +284,8 @@ stages: - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys1_mingw64_debug_openssl displayName: msys1 mingw64 debug openssl @@ -283,6 +305,8 @@ stages: - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys2_mingw32_debug_schannel displayName: msys2 mingw32 debug schannel @@ -304,6 +328,8 @@ stages: - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys2_mingw64_debug_schannel displayName: msys2 mingw64 debug schannel @@ -325,6 +351,8 @@ stages: - script: C:\msys64\usr\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys1_mingw_debug_schannel displayName: msys1 mingw debug schannel @@ -344,6 +372,8 @@ stages: - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys1_mingw32_debug_schannel displayName: msys1 mingw32 debug schannel @@ -363,6 +393,8 @@ stages: - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" - job: windows_msys1_mingw64_debug_schannel displayName: msys1 mingw64 debug schannel @@ -382,3 +414,5 @@ stages: - script: C:\MinGW\msys\1.0\bin\sh -l -c "cd $(echo '%cd%') && make test-nonflaky" displayName: 'test' + env: + AZURE_ACCESS_TOKEN: "$(System.AccessToken)" diff --git a/tests/azure.pm b/tests/azure.pm new file mode 100644 index 000000000..ef0bb6211 --- /dev/null +++ b/tests/azure.pm @@ -0,0 +1,140 @@ +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) 2020, Daniel Stenberg, <daniel@haxx.se>, et al. +# Copyright (C) 2020, Marc Hoersken, <info@marc-hoersken.de> +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.haxx.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +########################################################################### + +use strict; +use warnings; + +use POSIX qw(strftime); + +sub azure_check_environment { + if(defined $ENV{'AZURE_ACCESS_TOKEN'} && + defined $ENV{'AGENT_JOBNAME'} && $ENV{'BUILD_BUILDID'} && + defined $ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'} && + defined $ENV{'SYSTEM_TEAMPROJECTID'}) { + return 1; + } + return 0; +} + +sub azure_create_test_run { + my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}"; + my $azure_run=`curl --silent \\ + --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\ + --header "Content-Type: application/json" \\ + --data " + { + 'name': '$ENV{'AGENT_JOBNAME'}', + 'automated': true, + 'build': {'id': '$ENV{'BUILD_BUILDID'}'} + } + " \\ + "$azure_baseurl/_apis/test/runs?api-version=5.0"`; + if($azure_run =~ /"id":(\d+)/) { + return $1; + } + return ""; +} + +sub azure_create_test_result { + my ($azure_run_id, $testnum, $testname)=@_; + my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}"; + my $azure_result=`curl --silent \\ + --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\ + --header "Content-Type: application/json" \\ + --data " + [ + { + 'build': {'id': '$ENV{'BUILD_BUILDID'}'}, + 'testCase': {'id': $testnum}, + 'testCaseTitle': '$testname', + 'automatedTestName': 'curl.tests.$testnum', + 'outcome': 'InProgress' + } + ] + " \\ + "$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`; + if($azure_result =~ /\[\{"id":(\d+)/) { + return $1; + } + return ""; +} + +sub azure_update_test_result { + my ($azure_run_id, $azure_result_id, $testnum, $error, $start, $stop)=@_; + if(!defined $stop) { + $stop = $start; + } + my $azure_start = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $start; + my $azure_complete = strftime "%Y-%m-%dT%H:%M:%SZ", gmtime $stop; + my $azure_duration = sprintf("%.0f", ($stop-$start)*1000); + my $azure_outcome; + if($error < 0) { + $azure_outcome = 'Not executed'; + } + elsif(!$error) { + $azure_outcome = 'Passed'; + } + else { + $azure_outcome = 'Failed'; + } + my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}"; + my $azure_result=`curl --silent --request PATCH \\ + --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\ + --header "Content-Type: application/json" \\ + --data " + [ + { + 'id': $azure_result_id, + 'outcome': '$azure_outcome', + 'startedDate': '$azure_start', + 'completedDate': '$azure_complete', + 'durationInMs': $azure_duration + } + ] + " \\ + "$azure_baseurl/_apis/test/runs/$azure_run_id/results?api-version=5.0"`; + if($azure_result =~ /\[\{"id":(\d+)/) { + return $1; + } + return ""; +} + +sub azure_update_test_run { + my ($azure_run_id)=@_; + my $azure_baseurl="$ENV{'SYSTEM_TEAMFOUNDATIONCOLLECTIONURI'}$ENV{'SYSTEM_TEAMPROJECTID'}"; + my $azure_run=`curl --silent --request PATCH \\ + --header "Authorization: Bearer $ENV{'AZURE_ACCESS_TOKEN'}" \\ + --header "Content-Type: application/json" \\ + --data " + { + 'state': 'Completed' + } + " \\ + "$azure_baseurl/_apis/test/runs/$azure_run_id?api-version=5.0"`; + if($azure_run =~ /"id":(\d+)/) { + return $1; + } + return ""; +} + +1; diff --git a/tests/runtests.pl b/tests/runtests.pl index 2fcd35fbc..1bd402562 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -111,6 +111,7 @@ use pathhelp; require "getpart.pm"; # array functions require "valgrind.pm"; # valgrind report parser require "ftp.pm"; +require "azure.pm"; my $HOSTIP="127.0.0.1"; # address on which the test server listens my $HOST6IP="[::1]"; # address on which the test server listens @@ -327,6 +328,10 @@ my $tortalloc; my $shallow; my $randseed = 0; +# Azure Pipelines specific variables +my $AZURE_RUN_ID = 0; +my $AZURE_RESULT_ID = 0; + ####################################################################### # logmsg is our general message logging subroutine. # @@ -3732,6 +3737,10 @@ sub singletest { close(GDBCMD); } + if(azure_check_environment() && $AZURE_RUN_ID) { + $AZURE_RESULT_ID = azure_create_test_result($AZURE_RUN_ID, $testnum, $testname); + } + # timestamp starting of test command $timetoolini{$testnum} = Time::HiRes::time(); @@ -4346,7 +4355,6 @@ sub singletest { logmsg "PASS: $testnum - $testname\n"; } - return 0; } @@ -5505,6 +5513,15 @@ sub displaylogs { } ####################################################################### +# Setup Azure Pipelines Test Run (if running in Azure DevOps) +# + +if(azure_check_environment()) { + $AZURE_RUN_ID = azure_create_test_run(); + logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose); +} + +####################################################################### # The main test-loop # @@ -5524,6 +5541,12 @@ foreach $testnum (@at) { $count++; my $error = singletest($run_event_based, $testnum, $count, scalar(@at)); + + if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) { + $AZURE_RESULT_ID = azure_update_test_result($AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error, + $timeprepini{$testnum}, $timevrfyend{$testnum}); + } + if($error < 0) { # not a test we can run next; @@ -5553,6 +5576,14 @@ foreach $testnum (@at) { my $sofar = time() - $start; ####################################################################### +# Finish Azure Pipelines Test Run (if running in Azure DevOps) +# + +if(azure_check_environment() && $AZURE_RUN_ID) { + $AZURE_RUN_ID = azure_update_test_run($AZURE_RUN_ID); +} + +####################################################################### # Close command log # close(CMDLOG); |