CMake: fix runtests.pl with CMake, add new test targets

  * runtests.pl:
    - Fix out-of-tree build under CMake when srcdir is not set. Default
      srcdir to the location of runtests.pl.
    - Add a hack to allow CMake to use the TFLAGS option as documented
      in tests/README and used in scripts/travis/script.sh.
  * Bump CMake version to 3.2 for USES_TERMINAL, dropping Debian Jessie
    support (no one should care, it is already EOL.).
  * Remove CTest since it defines its own 'test' target with no tests
    since all unittests are already broken and not built by default.
  * Add new test targets based on the options from Makefile.am. Since
    new test targets are rarely added, I opted for duplicating the
    runtests.pl options as opposed to creating a new Makefile.inc file.
    Use top-level target names (test-x) instead of x-test since that is
    used by CI and others.

Closes #5358

5 files changed, 53 insertions, 31 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 224a09b1d..e9b90860c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -26,7 +26,6 @@
 # The output .so file lacks the soname number which we currently have within the lib/Makefile.am file
 # Add full (4 or 5 libs) SSL support
 # Add INSTALL target (EXTRA_DIST variables in Makefile.am may be moved to Makefile.inc so that CMake/CPack is aware of what's to include).
-# Add CTests(?)
 # Check on all possible platforms
 # Test with as many configurations possible (With or without any option)
 # Create scripts that help keeping the CMake build system up to date (to reduce maintenance). According to Tetetest:
@@ -38,7 +37,7 @@
 # To check:
 # (From Daniel Stenberg) The cmake build selected to run gcc with -fPIC on my box while the plain configure script did not.
 # (From Daniel Stenberg) The gcc command line use neither -g nor any -O options. As a developer, I also treasure our configure scripts's --enable-debug option that sets a long range of "picky" compiler options.
-cmake_minimum_required(VERSION 3.0...3.16 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.2...3.16 FATAL_ERROR)
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake;${CMAKE_MODULE_PATH}")
 include(Utilities)
@@ -221,10 +220,6 @@ cmake_dependent_option(ENABLE_MANUAL "to provide the built-in manual"
     ON "NROFF_USEFUL;PERL_FOUND"
     OFF)
 
-if(NOT PERL_FOUND)
-  message(STATUS "Perl not found, testing disabled.")
-  set(BUILD_TESTING OFF)
-endif()
 if(ENABLE_MANUAL)
   set(USE_MANUAL ON)
 endif()
@@ -1280,8 +1275,10 @@ if(BUILD_CURL_EXE)
   add_subdirectory(src)
 endif()
 
-include(CTest)
-if(BUILD_TESTING)
+option(BUILD_TESTING "Build tests" "${PERL_FOUND}")
+if(NOT PERL_FOUND)
+  message(STATUS "Perl not found, testing disabled.")
+elseif(BUILD_TESTING)
   add_subdirectory(tests)
 endif()
 
diff --git a/docs/TODO b/docs/TODO
index ce92b21d9..68b8faa62 100644
--- a/docs/TODO
+++ b/docs/TODO
@@ -1071,14 +1071,6 @@ that doesn't exist on the server, just like --ftp-create-dirs.
  to no impact, neither on the performance nor on the general functionality of
  curl.
 
-19.3 cmake test suite improvements
-
- The cmake build doesn't support 'make show' so it doesn't know which tests
- are in the makefile or not (making appveyor builds do many false warnings
- about it) nor does it support running the test suite if building out-of-tree.
-
- See https://github.com/curl/curl/issues/3109
-
 20. Test suite
 
 20.1 SSL tunnel
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index caba3c86a..60ce5c262 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -24,3 +24,28 @@ add_subdirectory(data)
 add_subdirectory(libtest)
 add_subdirectory(server)
 add_subdirectory(unit)
+
+function(add_runtests targetname test_flags)
+  # Use a special '${TFLAGS}' placeholder as last argument which will be
+  # replaced by the contents of the environment variable in runtests.pl.
+  # This is a workaround for CMake's limitation where commands executed by
+  # 'make' or 'ninja' cannot portably reference environment variables.
+  string(REPLACE " " ";" test_flags_list "${test_flags}")
+  add_custom_target(${targetname}
+    COMMAND
+      "${PERL_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/runtests.pl"
+      ${test_flags_list}
+      "\${TFLAGS}"
+    DEPENDS testdeps
+    VERBATIM USES_TERMINAL
+  )
+endfunction()
+
+add_runtests(test           "")
+add_runtests(test-quiet     "-a -s")
+add_runtests(test-am        "-a -am")
+add_runtests(test-full      "-a -p -r")
+# !flaky means that it'll skip all tests using the flaky keyword
+add_runtests(test-nonflaky  "-a -p !flaky")
+add_runtests(test-torture   "-a -t")
+add_runtests(test-event     "-a -e")
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 33334a751..4eac05f90 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -56,8 +56,14 @@
 # These should be the only variables that might be needed to get edited:
 
 BEGIN {
-    push(@INC, $ENV{'srcdir'}) if(defined $ENV{'srcdir'});
-    push(@INC, ".");
+    # Define srcdir to the location of the tests source directory. This is
+    # usually set by the Makefile, but for out-of-tree builds with direct
+    # invocation of runtests.pl, it may not be set.
+    if(!defined $ENV{'srcdir'}) {
+        use File::Basename;
+        $ENV{'srcdir'} = dirname(__FILE__);
+    }
+    push(@INC, $ENV{'srcdir'});
     # run time statistics needs Time::HiRes
     eval {
         no warnings "all";
@@ -559,7 +565,11 @@ sub checkcmd {
 #
 my $disttests;
 sub get_disttests {
-    my @dist = `cd data && make show`;
+    my $makeCmd = 'make';
+    if(-f "../CMakeCache.txt") {
+        $makeCmd = 'cmake --build ../.. --target';
+    }
+    my @dist = `cd data && $makeCmd show`;
     $disttests = join("", @dist);
 }
 
@@ -5120,6 +5130,13 @@ disabledtests("$TESTDIR/DISABLED.local");
 # Check options to this test program
 #
 
+# Special case for CMake: replace '${TFLAGS}' by the contents of the
+# environment variable (if any).
+if(@ARGV && $ARGV[-1] eq '${TFLAGS}') {
+    pop @ARGV;
+    push(@ARGV, split(' ', $ENV{'TFLAGS'})) if defined($ENV{'TFLAGS'});
+}
+
 my $number=0;
 my $fromnum=-1;
 my @testthis;
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index a18ea18e0..e5c4127a0 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -20,6 +20,9 @@
 #
 ###########################################################################
 
+# TODO build a special libcurlu library for unittests.
+return()
+
 set(UT_SRC
   unit1300.c
   unit1301.c
@@ -63,16 +66,4 @@ foreach(_testfile ${UT_SRC})
   target_link_libraries(${_testname} libcurl ${CURL_LIBS})
   set_target_properties(${_testname}
       PROPERTIES COMPILE_DEFINITIONS "UNITTESTS")
-
-  if(HIDES_CURL_PRIVATE_SYMBOLS)
-    set_target_properties(${_testname}
-      PROPERTIES
-      EXCLUDE_FROM_ALL TRUE
-      EXCLUDE_FROM_DEFAULT_BUILD TRUE
-    )
-  else()
-    add_test(NAME ${_testname}
-             COMMAND ${_testname} "http://www.google.com"
-    )
-  endif()
 endforeach()