|
| 1 | +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying |
| 2 | +# file Copyright.txt or https://cmake.org/licensing for details. |
| 3 | + |
| 4 | +#[=======================================================================[.rst: |
| 5 | +Catch |
| 6 | +----- |
| 7 | +
|
| 8 | +This module defines a function to help use the Catch test framework. |
| 9 | +
|
| 10 | +The :command:`catch_discover_tests` discovers tests by asking the compiled test |
| 11 | +executable to enumerate its tests. This does not require CMake to be re-run |
| 12 | +when tests change. However, it may not work in a cross-compiling environment, |
| 13 | +and setting test properties is less convenient. |
| 14 | +
|
| 15 | +This command is intended to replace use of :command:`add_test` to register |
| 16 | +tests, and will create a separate CTest test for each Catch test case. Note |
| 17 | +that this is in some cases less efficient, as common set-up and tear-down logic |
| 18 | +cannot be shared by multiple test cases executing in the same instance. |
| 19 | +However, it provides more fine-grained pass/fail information to CTest, which is |
| 20 | +usually considered as more beneficial. By default, the CTest test name is the |
| 21 | +same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. |
| 22 | +
|
| 23 | +.. command:: catch_discover_tests |
| 24 | +
|
| 25 | + Automatically add tests with CTest by querying the compiled test executable |
| 26 | + for available tests:: |
| 27 | +
|
| 28 | + catch_discover_tests(target |
| 29 | + [TEST_SPEC arg1...] |
| 30 | + [EXTRA_ARGS arg1...] |
| 31 | + [WORKING_DIRECTORY dir] |
| 32 | + [TEST_PREFIX prefix] |
| 33 | + [TEST_SUFFIX suffix] |
| 34 | + [PROPERTIES name1 value1...] |
| 35 | + [TEST_LIST var] |
| 36 | + [REPORTER reporter] |
| 37 | + [OUTPUT_DIR dir] |
| 38 | + [OUTPUT_PREFIX prefix] |
| 39 | + [OUTPUT_SUFFIX suffix] |
| 40 | + [DISCOVERY_MODE <POST_BUILD|PRE_TEST>] |
| 41 | + ) |
| 42 | +
|
| 43 | + ``catch_discover_tests`` sets up a post-build command on the test executable |
| 44 | + that generates the list of tests by parsing the output from running the test |
| 45 | + with the ``--list-test-names-only`` argument. This ensures that the full |
| 46 | + list of tests is obtained. Since test discovery occurs at build time, it is |
| 47 | + not necessary to re-run CMake when the list of tests changes. |
| 48 | + However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set |
| 49 | + in order to function in a cross-compiling environment. |
| 50 | +
|
| 51 | + Additionally, setting properties on tests is somewhat less convenient, since |
| 52 | + the tests are not available at CMake time. Additional test properties may be |
| 53 | + assigned to the set of tests as a whole using the ``PROPERTIES`` option. If |
| 54 | + more fine-grained test control is needed, custom content may be provided |
| 55 | + through an external CTest script using the :prop_dir:`TEST_INCLUDE_FILES` |
| 56 | + directory property. The set of discovered tests is made accessible to such a |
| 57 | + script via the ``<target>_TESTS`` variable. |
| 58 | +
|
| 59 | + The options are: |
| 60 | +
|
| 61 | + ``target`` |
| 62 | + Specifies the Catch executable, which must be a known CMake executable |
| 63 | + target. CMake will substitute the location of the built executable when |
| 64 | + running the test. |
| 65 | +
|
| 66 | + ``TEST_SPEC arg1...`` |
| 67 | + Specifies test cases, wildcarded test cases, tags and tag expressions to |
| 68 | + pass to the Catch executable with the ``--list-test-names-only`` argument. |
| 69 | +
|
| 70 | + ``EXTRA_ARGS arg1...`` |
| 71 | + Any extra arguments to pass on the command line to each test case. |
| 72 | +
|
| 73 | + ``WORKING_DIRECTORY dir`` |
| 74 | + Specifies the directory in which to run the discovered test cases. If this |
| 75 | + option is not provided, the current binary directory is used. |
| 76 | +
|
| 77 | + ``TEST_PREFIX prefix`` |
| 78 | + Specifies a ``prefix`` to be prepended to the name of each discovered test |
| 79 | + case. This can be useful when the same test executable is being used in |
| 80 | + multiple calls to ``catch_discover_tests()`` but with different |
| 81 | + ``TEST_SPEC`` or ``EXTRA_ARGS``. |
| 82 | +
|
| 83 | + ``TEST_SUFFIX suffix`` |
| 84 | + Similar to ``TEST_PREFIX`` except the ``suffix`` is appended to the name of |
| 85 | + every discovered test case. Both ``TEST_PREFIX`` and ``TEST_SUFFIX`` may |
| 86 | + be specified. |
| 87 | +
|
| 88 | + ``PROPERTIES name1 value1...`` |
| 89 | + Specifies additional properties to be set on all tests discovered by this |
| 90 | + invocation of ``catch_discover_tests``. |
| 91 | +
|
| 92 | + ``TEST_LIST var`` |
| 93 | + Make the list of tests available in the variable ``var``, rather than the |
| 94 | + default ``<target>_TESTS``. This can be useful when the same test |
| 95 | + executable is being used in multiple calls to ``catch_discover_tests()``. |
| 96 | + Note that this variable is only available in CTest. |
| 97 | +
|
| 98 | + ``REPORTER reporter`` |
| 99 | + Use the specified reporter when running the test case. The reporter will |
| 100 | + be passed to the Catch executable as ``--reporter reporter``. |
| 101 | +
|
| 102 | + ``OUTPUT_DIR dir`` |
| 103 | + If specified, the parameter is passed along as |
| 104 | + ``--out dir/<test_name>`` to Catch executable. The actual file name is the |
| 105 | + same as the test name. This should be used instead of |
| 106 | + ``EXTRA_ARGS --out foo`` to avoid race conditions writing the result output |
| 107 | + when using parallel test execution. |
| 108 | +
|
| 109 | + ``OUTPUT_PREFIX prefix`` |
| 110 | + May be used in conjunction with ``OUTPUT_DIR``. |
| 111 | + If specified, ``prefix`` is added to each output file name, like so |
| 112 | + ``--out dir/prefix<test_name>``. |
| 113 | +
|
| 114 | + ``OUTPUT_SUFFIX suffix`` |
| 115 | + May be used in conjunction with ``OUTPUT_DIR``. |
| 116 | + If specified, ``suffix`` is added to each output file name, like so |
| 117 | + ``--out dir/<test_name>suffix``. This can be used to add a file extension to |
| 118 | + the output e.g. ".xml". |
| 119 | +
|
| 120 | + ``DL_PATHS path...`` |
| 121 | + Specifies paths that need to be set for the dynamic linker to find shared |
| 122 | + libraries/DLLs when running the test executable (PATH/LD_LIBRARY_PATH respectively). |
| 123 | + These paths will both be set when retrieving the list of test cases from the |
| 124 | + test executable and when the tests are executed themselves. This requires |
| 125 | + cmake/ctest >= 3.22. |
| 126 | +
|
| 127 | + ``DL_FRAMEWORK_PATHS path...`` |
| 128 | + Specifies paths that need to be set for the dynamic linker to find libraries |
| 129 | + packaged as frameworks on Apple platforms when running the test executable |
| 130 | + (DYLD_FRAMEWORK_PATH). These paths will both be set when retrieving the list |
| 131 | + of test cases from the test executable and when the tests are executed themselves. |
| 132 | + This requires cmake/ctest >= 3.22. |
| 133 | +
|
| 134 | + `DISCOVERY_MODE mode`` |
| 135 | + Provides control over when ``catch_discover_tests`` performs test discovery. |
| 136 | + By default, ``POST_BUILD`` sets up a post-build command to perform test discovery |
| 137 | + at build time. In certain scenarios, like cross-compiling, this ``POST_BUILD`` |
| 138 | + behavior is not desirable. By contrast, ``PRE_TEST`` delays test discovery until |
| 139 | + just prior to test execution. This way test discovery occurs in the target environment |
| 140 | + where the test has a better chance at finding appropriate runtime dependencies. |
| 141 | +
|
| 142 | + ``DISCOVERY_MODE`` defaults to the value of the |
| 143 | + ``CMAKE_CATCH_DISCOVER_TESTS_DISCOVERY_MODE`` variable if it is not passed when |
| 144 | + calling ``catch_discover_tests``. This provides a mechanism for globally selecting |
| 145 | + a preferred test discovery behavior without having to modify each call site. |
| 146 | +
|
| 147 | +#]=======================================================================] |
| 148 | + |
| 149 | +#------------------------------------------------------------------------------ |
| 150 | +function(catch_discover_tests TARGET) |
| 151 | + |
| 152 | + cmake_parse_arguments( |
| 153 | + "" |
| 154 | + "" |
| 155 | + "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE" |
| 156 | + "TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS" |
| 157 | + ${ARGN} |
| 158 | + ) |
| 159 | + |
| 160 | + if(NOT _WORKING_DIRECTORY) |
| 161 | + set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") |
| 162 | + endif() |
| 163 | + if(NOT _TEST_LIST) |
| 164 | + set(_TEST_LIST ${TARGET}_TESTS) |
| 165 | + endif() |
| 166 | + if(_DL_PATHS AND ${CMAKE_VERSION} VERSION_LESS "3.22.0") |
| 167 | + message(FATAL_ERROR "The DL_PATHS option requires at least cmake 3.22") |
| 168 | + endif() |
| 169 | + if(_DL_FRAMEWORK_PATHS AND ${CMAKE_VERSION} VERSION_LESS "3.22.0") |
| 170 | + message(FATAL_ERROR "The DL_FRAMEWORK_PATHS option requires at least cmake 3.22") |
| 171 | + endif() |
| 172 | + if(NOT _DISCOVERY_MODE) |
| 173 | + if(NOT CMAKE_CATCH_DISCOVER_TESTS_DISCOVERY_MODE) |
| 174 | + set(CMAKE_CATCH_DISCOVER_TESTS_DISCOVERY_MODE "POST_BUILD") |
| 175 | + endif() |
| 176 | + set(_DISCOVERY_MODE ${CMAKE_CATCH_DISCOVER_TESTS_DISCOVERY_MODE}) |
| 177 | + endif() |
| 178 | + if (NOT _DISCOVERY_MODE MATCHES "^(POST_BUILD|PRE_TEST)$") |
| 179 | + message(FATAL_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}") |
| 180 | + endif() |
| 181 | + |
| 182 | + ## Generate a unique name based on the extra arguments |
| 183 | + string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}") |
| 184 | + string(SUBSTRING ${args_hash} 0 7 args_hash) |
| 185 | + |
| 186 | + # Define rule to generate test list for aforementioned test executable |
| 187 | + set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-${args_hash}") |
| 188 | + set(ctest_include_file "${ctest_file_base}_include.cmake") |
| 189 | + set(ctest_tests_file "${ctest_file_base}_tests.cmake") |
| 190 | + |
| 191 | + get_property(crosscompiling_emulator |
| 192 | + TARGET ${TARGET} |
| 193 | + PROPERTY CROSSCOMPILING_EMULATOR |
| 194 | + ) |
| 195 | + |
| 196 | + if(_DISCOVERY_MODE STREQUAL "POST_BUILD") |
| 197 | + add_custom_command( |
| 198 | + TARGET ${TARGET} POST_BUILD |
| 199 | + BYPRODUCTS "${ctest_tests_file}" |
| 200 | + COMMAND "${CMAKE_COMMAND}" |
| 201 | + -D "TEST_TARGET=${TARGET}" |
| 202 | + -D "TEST_EXECUTABLE=$<TARGET_FILE:${TARGET}>" |
| 203 | + -D "TEST_EXECUTOR=${crosscompiling_emulator}" |
| 204 | + -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" |
| 205 | + -D "TEST_SPEC=${_TEST_SPEC}" |
| 206 | + -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" |
| 207 | + -D "TEST_PROPERTIES=${_PROPERTIES}" |
| 208 | + -D "TEST_PREFIX=${_TEST_PREFIX}" |
| 209 | + -D "TEST_SUFFIX=${_TEST_SUFFIX}" |
| 210 | + -D "TEST_LIST=${_TEST_LIST}" |
| 211 | + -D "TEST_REPORTER=${_REPORTER}" |
| 212 | + -D "TEST_OUTPUT_DIR=${_OUTPUT_DIR}" |
| 213 | + -D "TEST_OUTPUT_PREFIX=${_OUTPUT_PREFIX}" |
| 214 | + -D "TEST_OUTPUT_SUFFIX=${_OUTPUT_SUFFIX}" |
| 215 | + -D "TEST_DL_PATHS=${_DL_PATHS}" |
| 216 | + -D "TEST_DL_FRAMEWORK_PATHS=${_DL_FRAMEWORK_PATHS}" |
| 217 | + -D "CTEST_FILE=${ctest_tests_file}" |
| 218 | + -P "${_CATCH_DISCOVER_TESTS_SCRIPT}" |
| 219 | + VERBATIM |
| 220 | + ) |
| 221 | + |
| 222 | + file(WRITE "${ctest_include_file}" |
| 223 | + "if(EXISTS \"${ctest_tests_file}\")\n" |
| 224 | + " include(\"${ctest_tests_file}\")\n" |
| 225 | + "else()\n" |
| 226 | + " add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n" |
| 227 | + "endif()\n" |
| 228 | + ) |
| 229 | + |
| 230 | + elseif(_DISCOVERY_MODE STREQUAL "PRE_TEST") |
| 231 | + |
| 232 | + get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL |
| 233 | + PROPERTY GENERATOR_IS_MULTI_CONFIG |
| 234 | + ) |
| 235 | + |
| 236 | + if(GENERATOR_IS_MULTI_CONFIG) |
| 237 | + set(ctest_tests_file "${ctest_file_base}_tests-$<CONFIG>.cmake") |
| 238 | + endif() |
| 239 | + |
| 240 | + string(CONCAT ctest_include_content |
| 241 | + "if(EXISTS \"$<TARGET_FILE:${TARGET}>\")" "\n" |
| 242 | + " if(NOT EXISTS \"${ctest_tests_file}\" OR" "\n" |
| 243 | + " NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"$<TARGET_FILE:${TARGET}>\" OR\n" |
| 244 | + " NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n" |
| 245 | + " include(\"${_CATCH_DISCOVER_TESTS_SCRIPT}\")" "\n" |
| 246 | + " catch_discover_tests_impl(" "\n" |
| 247 | + " TEST_EXECUTABLE" " [==[" "$<TARGET_FILE:${TARGET}>" "]==]" "\n" |
| 248 | + " TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n" |
| 249 | + " TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n" |
| 250 | + " TEST_SPEC" " [==[" "${_TEST_SPEC}" "]==]" "\n" |
| 251 | + " TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n" |
| 252 | + " TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n" |
| 253 | + " TEST_PREFIX" " [==[" "${_TEST_PREFIX}" "]==]" "\n" |
| 254 | + " TEST_SUFFIX" " [==[" "${_TEST_SUFFIX}" "]==]" "\n" |
| 255 | + " TEST_LIST" " [==[" "${_TEST_LIST}" "]==]" "\n" |
| 256 | + " TEST_REPORTER" " [==[" "${_REPORTER}" "]==]" "\n" |
| 257 | + " TEST_OUTPUT_DIR" " [==[" "${_OUTPUT_DIR}" "]==]" "\n" |
| 258 | + " TEST_OUTPUT_PREFIX" " [==[" "${_OUTPUT_PREFIX}" "]==]" "\n" |
| 259 | + " TEST_OUTPUT_SUFFIX" " [==[" "${_OUTPUT_SUFFIX}" "]==]" "\n" |
| 260 | + " CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n" |
| 261 | + " TEST_DL_PATHS" " [==[" "${_DL_PATHS}" "]==]" "\n" |
| 262 | + " TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}" "]==]" "\n" |
| 263 | + " CTEST_FILE" " [==[" "${CTEST_FILE}" "]==]" "\n" |
| 264 | + " )" "\n" |
| 265 | + " endif()" "\n" |
| 266 | + " include(\"${ctest_tests_file}\")" "\n" |
| 267 | + "else()" "\n" |
| 268 | + " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n" |
| 269 | + "endif()" "\n" |
| 270 | + ) |
| 271 | + |
| 272 | + if(GENERATOR_IS_MULTI_CONFIG) |
| 273 | + foreach(_config ${CMAKE_CONFIGURATION_TYPES}) |
| 274 | + file(GENERATE OUTPUT "${ctest_file_base}_include-${_config}.cmake" CONTENT "${ctest_include_content}" CONDITION $<CONFIG:${_config}>) |
| 275 | + endforeach() |
| 276 | + string(CONCAT ctest_include_multi_content |
| 277 | + "if(NOT CTEST_CONFIGURATION_TYPE)" "\n" |
| 278 | + " message(\"No configuration for testing specified, use '-C <cfg>'.\")" "\n" |
| 279 | + "else()" "\n" |
| 280 | + " include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")" "\n" |
| 281 | + "endif()" "\n" |
| 282 | + ) |
| 283 | + file(GENERATE OUTPUT "${ctest_include_file}" CONTENT "${ctest_include_multi_content}") |
| 284 | + else() |
| 285 | + file(GENERATE OUTPUT "${ctest_file_base}_include.cmake" CONTENT "${ctest_include_content}") |
| 286 | + file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include.cmake\")") |
| 287 | + endif() |
| 288 | + endif() |
| 289 | + |
| 290 | + if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0") |
| 291 | + # Add discovered tests to directory TEST_INCLUDE_FILES |
| 292 | + set_property(DIRECTORY |
| 293 | + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" |
| 294 | + ) |
| 295 | + else() |
| 296 | + # Add discovered tests as directory TEST_INCLUDE_FILE if possible |
| 297 | + get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET) |
| 298 | + if (NOT ${test_include_file_set}) |
| 299 | + set_property(DIRECTORY |
| 300 | + PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}" |
| 301 | + ) |
| 302 | + else() |
| 303 | + message(FATAL_ERROR "Cannot set more than one TEST_INCLUDE_FILE") |
| 304 | + endif() |
| 305 | + endif() |
| 306 | + |
| 307 | +endfunction() |
| 308 | + |
| 309 | +############################################################################### |
| 310 | + |
| 311 | +set(_CATCH_DISCOVER_TESTS_SCRIPT |
| 312 | + ${CMAKE_CURRENT_LIST_DIR}/CatchAddTests.cmake |
| 313 | + CACHE INTERNAL "Catch2 full path to CatchAddTests.cmake helper file" |
| 314 | +) |
0 commit comments