Skip to content

Commit c15c563

Browse files
NgoQuocViet2001NgoQuocViet2001
committed
Fix relative PRE_TEST discovery paths
Closes #3051
1 parent 0aeb818 commit c15c563

5 files changed

Lines changed: 181 additions & 39 deletions

File tree

‎docs/cmake-integration.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ catch_discover_tests(target
127127
[OUTPUT_PREFIX prefix]
128128
[OUTPUT_SUFFIX suffix]
129129
[DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
130+
[USE_RELATIVE_PATHS]
130131
[SKIP_IS_FAILURE]
131132
[ADD_TAGS_AS_LABELS]
132133
)
@@ -223,6 +224,12 @@ limitation affects CMake's `gtest_discover_tests`; see
223224
[Catch2 #2411](https://github.com/catchorg/Catch2/issues/2411) and
224225
[CMake #21845](https://gitlab.kitware.com/cmake/cmake/-/issues/21845)._
225226

227+
* `USE_RELATIVE_PATHS`
228+
229+
With `DISCOVERY_MODE PRE_TEST`, stores generated discovery paths relative to
230+
the CTest include file. This allows the build tree and test executable to be
231+
relocated before test discovery runs. This option requires CMake 3.24 or newer.
232+
226233
* `SKIP_IS_FAILURE`
227234

228235
Skipped tests will be marked as failed instead.

‎extras/Catch.cmake‎

Lines changed: 79 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
3838
[OUTPUT_PREFIX prefix]
3939
[OUTPUT_SUFFIX suffix]
4040
[DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
41+
[USE_RELATIVE_PATHS]
4142
[SKIP_IS_FAILURE]
4243
[ADD_TAGS_AS_LABELS]
4344
)
@@ -157,6 +158,12 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
157158
code-signs the test executable only after the post-build script that
158159
``POST_BUILD`` mode uses to run it for test discovery. See Catch2 issue #2411.
159160
161+
``USE_RELATIVE_PATHS``
162+
Makes the files generated for ``PRE_TEST`` discovery use paths relative to
163+
the generated CTest include file. This allows a build tree to be relocated
164+
with its test executable before test discovery runs. This option requires
165+
CMake 3.24 or newer.
166+
160167
``SKIP_IS_FAILURE``
161168
Disables skipped test detection.
162169
@@ -170,7 +177,7 @@ function(catch_discover_tests TARGET)
170177

171178
cmake_parse_arguments(
172179
""
173-
"SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS"
180+
"SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS;USE_RELATIVE_PATHS"
174181
"TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE"
175182
"TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS"
176183
${ARGN}
@@ -201,6 +208,12 @@ function(catch_discover_tests TARGET)
201208
if(NOT _DISCOVERY_MODE MATCHES "^(POST_BUILD|PRE_TEST)$")
202209
message(FATAL_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}")
203210
endif()
211+
if(_USE_RELATIVE_PATHS AND NOT _DISCOVERY_MODE STREQUAL "PRE_TEST")
212+
message(FATAL_ERROR "USE_RELATIVE_PATHS requires DISCOVERY_MODE PRE_TEST")
213+
endif()
214+
if(_USE_RELATIVE_PATHS AND CMAKE_VERSION VERSION_LESS "3.24")
215+
message(FATAL_ERROR "USE_RELATIVE_PATHS requires CMake 3.24 or newer")
216+
endif()
204217

205218
## Generate a unique name based on the extra arguments
206219
string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}")
@@ -210,6 +223,7 @@ function(catch_discover_tests TARGET)
210223
set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-${args_hash}")
211224
set(ctest_include_file "${ctest_file_base}_include.cmake")
212225
set(ctest_tests_file "${ctest_file_base}_tests.cmake")
226+
set(ctest_include_file_for_property "${ctest_include_file}")
213227

214228
get_property(crosscompiling_emulator
215229
TARGET ${TARGET}
@@ -264,16 +278,69 @@ function(catch_discover_tests TARGET)
264278
set(ctest_tests_file "${ctest_file_base}_tests-$<CONFIG>.cmake")
265279
endif()
266280

281+
set(test_executable_for_script "$<TARGET_FILE:${TARGET}>")
282+
set(test_working_dir_for_script "${_WORKING_DIRECTORY}")
283+
set(discover_tests_script_for_script "${_CATCH_DISCOVER_TESTS_SCRIPT}")
284+
set(ctest_tests_file_for_script "${ctest_tests_file}")
285+
set(test_executable_argument "[==[${test_executable_for_script}]==]")
286+
set(test_working_dir_argument "[==[${test_working_dir_for_script}]==]")
287+
set(ctest_file_argument "[==[${ctest_tests_file_for_script}]==]")
288+
set(ctest_path_setup)
289+
set(ctest_config_include_file
290+
"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake"
291+
)
292+
293+
if(_USE_RELATIVE_PATHS)
294+
get_filename_component(ctest_file_base_name "${ctest_file_base}" NAME)
295+
get_filename_component(ctest_tests_file_name "${ctest_tests_file}" NAME)
296+
get_filename_component(ctest_include_file_name "${ctest_include_file}" NAME)
297+
298+
set(test_executable_relative
299+
"$<PATH:RELATIVE_PATH,$<TARGET_FILE:${TARGET}>,${CMAKE_CURRENT_BINARY_DIR}>"
300+
)
301+
set(test_working_dir_relative
302+
"$<PATH:RELATIVE_PATH,$<PATH:ABSOLUTE_PATH,${_WORKING_DIRECTORY},${CMAKE_CURRENT_BINARY_DIR}>,${CMAKE_CURRENT_BINARY_DIR}>"
303+
)
304+
set(discover_tests_script_name
305+
"${ctest_file_base_name}_CatchAddTests.cmake"
306+
)
307+
configure_file(
308+
"${_CATCH_DISCOVER_TESTS_SCRIPT}"
309+
"${CMAKE_CURRENT_BINARY_DIR}/${discover_tests_script_name}"
310+
COPYONLY
311+
)
312+
313+
set(ctest_config_include_file
314+
"${ctest_file_base_name}_include-\${CTEST_CONFIGURATION_TYPE}.cmake"
315+
)
316+
string(CONCAT ctest_path_setup
317+
"get_filename_component(_catch_discover_tests_script \"${discover_tests_script_name}\" REALPATH)" "\n"
318+
"get_filename_component(_catch_discovery_dir \"\${_catch_discover_tests_script}\" DIRECTORY)" "\n"
319+
"set(_catch_test_executable \"\${_catch_discovery_dir}/${test_executable_relative}\")" "\n"
320+
"set(_catch_test_working_dir \"\${_catch_discovery_dir}/${test_working_dir_relative}\")" "\n"
321+
"set(_catch_tests_file \"\${_catch_discovery_dir}/${ctest_tests_file_name}\")" "\n"
322+
)
323+
set(test_executable_for_script "\${_catch_test_executable}")
324+
set(test_working_dir_for_script "\${_catch_test_working_dir}")
325+
set(discover_tests_script_for_script "\${_catch_discover_tests_script}")
326+
set(ctest_tests_file_for_script "\${_catch_tests_file}")
327+
set(test_executable_argument "\"\${_catch_test_executable}\"")
328+
set(test_working_dir_argument "\"\${_catch_test_working_dir}\"")
329+
set(ctest_file_argument "\"\${_catch_tests_file}\"")
330+
set(ctest_include_file_for_property "${ctest_include_file_name}")
331+
endif()
332+
267333
string(CONCAT ctest_include_content
268-
"if(EXISTS \"$<TARGET_FILE:${TARGET}>\")" "\n"
269-
" if(NOT EXISTS \"${ctest_tests_file}\" OR" "\n"
270-
" NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"$<TARGET_FILE:${TARGET}>\" OR\n"
271-
" NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n"
272-
" include(\"${_CATCH_DISCOVER_TESTS_SCRIPT}\")" "\n"
334+
"${ctest_path_setup}"
335+
"if(EXISTS \"${test_executable_for_script}\")" "\n"
336+
" if(NOT EXISTS \"${ctest_tests_file_for_script}\" OR" "\n"
337+
" NOT \"${ctest_tests_file_for_script}\" IS_NEWER_THAN \"${test_executable_for_script}\" OR\n"
338+
" NOT \"${ctest_tests_file_for_script}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n"
339+
" include(\"${discover_tests_script_for_script}\")" "\n"
273340
" catch_discover_tests_impl(" "\n"
274-
" TEST_EXECUTABLE" " [==[" "$<TARGET_FILE:${TARGET}>" "]==]" "\n"
341+
" TEST_EXECUTABLE" " ${test_executable_argument}" "\n"
275342
" TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n"
276-
" TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n"
343+
" TEST_WORKING_DIR" " ${test_working_dir_argument}" "\n"
277344
" TEST_SPEC" " [==[" "${_TEST_SPEC}" "]==]" "\n"
278345
" TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n"
279346
" TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n"
@@ -284,13 +351,13 @@ function(catch_discover_tests TARGET)
284351
" TEST_OUTPUT_DIR" " [==[" "${_OUTPUT_DIR}" "]==]" "\n"
285352
" TEST_OUTPUT_PREFIX" " [==[" "${_OUTPUT_PREFIX}" "]==]" "\n"
286353
" TEST_OUTPUT_SUFFIX" " [==[" "${_OUTPUT_SUFFIX}" "]==]" "\n"
287-
" CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n"
354+
" CTEST_FILE" " ${ctest_file_argument}" "\n"
288355
" TEST_DL_PATHS" " [==[" "${_DL_PATHS}" "]==]" "\n"
289356
" TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}" "]==]" "\n"
290357
" ADD_TAGS_AS_LABELS" " [==[" "${_ADD_TAGS_AS_LABELS}" "]==]" "\n"
291358
" )" "\n"
292359
" endif()" "\n"
293-
" include(\"${ctest_tests_file}\")" "\n"
360+
" include(\"${ctest_tests_file_for_script}\")" "\n"
294361
"else()" "\n"
295362
" add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n"
296363
"endif()" "\n"
@@ -304,7 +371,7 @@ function(catch_discover_tests TARGET)
304371
"if(NOT CTEST_CONFIGURATION_TYPE)" "\n"
305372
" message(\"No configuration for testing specified, use '-C <cfg>'.\")" "\n"
306373
"else()" "\n"
307-
" include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")" "\n"
374+
" include(\"${ctest_config_include_file}\")" "\n"
308375
"endif()" "\n"
309376
)
310377
file(GENERATE OUTPUT "${ctest_include_file}" CONTENT "${ctest_include_multi_content}")
@@ -316,7 +383,7 @@ function(catch_discover_tests TARGET)
316383

317384
# Add discovered tests to directory TEST_INCLUDE_FILES
318385
set_property(DIRECTORY
319-
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
386+
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file_for_property}"
320387
)
321388

322389
endfunction()

‎tests/TestScripts/DiscoverTests/CMakeLists.txt‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,15 @@ if(CMAKE_VERSION GREATER_EQUAL 3.27)
1313
DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
1414
)
1515
endif()
16-
catch_discover_tests(
17-
tests
18-
ADD_TAGS_AS_LABELS
19-
DISCOVERY_MODE PRE_TEST
20-
${extra_args}
21-
)
16+
if(CATCH2_TEST_USE_RELATIVE_PATHS)
17+
add_subdirectory(relative-registration)
18+
else()
19+
catch_discover_tests(
20+
tests
21+
ADD_TAGS_AS_LABELS
22+
DISCOVERY_MODE PRE_TEST
23+
${extra_args}
24+
)
25+
endif()
2226

2327
# DISCOVERY_MODE <POST_BUILD|PRE_TEST>

‎tests/TestScripts/DiscoverTests/VerifyRegistration.py‎

Lines changed: 77 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,17 @@ def get_cmake_version():
3535
int(version_match.group(2)),
3636
int(version_match.group(3)))
3737

38-
def build_project(sources_dir, output_base_path, catch2_path):
38+
def build_project(sources_dir, output_base_path, catch2_path,
39+
use_relative_paths=False):
3940
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
4041
config_cmd = ['cmake',
4142
'-B', build_dir,
4243
'-S', sources_dir,
4344
f'-DCATCH2_PATH={catch2_path}',
4445
'-DCMAKE_BUILD_TYPE=Debug']
46+
relative_paths_setting = 'ON' if use_relative_paths else 'OFF'
47+
config_cmd.append(
48+
f'-DCATCH2_TEST_USE_RELATIVE_PATHS={relative_paths_setting}')
4549

4650
build_cmd = ['cmake',
4751
'--build', build_dir,
@@ -100,23 +104,50 @@ def get_test_names(build_path: str) -> List[TestInfo]:
100104

101105
def get_ctest_listing(build_path):
102106
old_path = os.getcwd()
103-
os.chdir(build_path)
107+
try:
108+
os.chdir(build_path)
109+
110+
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
111+
try:
112+
result = subprocess.run(cmd,
113+
capture_output = True,
114+
check = True,
115+
text = True)
116+
except subprocess.CalledProcessError as err:
117+
print('Error when getting output from CTest')
118+
print(f'cmd: {err.cmd}')
119+
print(f'stderr: {err.stderr}')
120+
print(f'stdout: {err.stdout}')
121+
exit(4)
122+
finally:
123+
os.chdir(old_path)
124+
125+
return result.stdout
104126

105-
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
127+
def run_ctest(build_path):
128+
cmd = ['ctest', '-C', 'debug', '--output-on-failure']
106129
try:
107-
result = subprocess.run(cmd,
108-
capture_output = True,
109-
check = True,
110-
text = True)
130+
subprocess.run(cmd,
131+
capture_output = True,
132+
check = True,
133+
cwd = build_path,
134+
text = True)
111135
except subprocess.CalledProcessError as err:
112-
print('Error when getting output from CTest')
136+
print('Error when running discovered tests')
113137
print(f'cmd: {err.cmd}')
114138
print(f'stderr: {err.stderr}')
115139
print(f'stdout: {err.stdout}')
116140
exit(4)
117141

118-
os.chdir(old_path)
119-
return result.stdout
142+
def remove_discovery_cache(build_path):
143+
relative_registration_dir = os.path.join(
144+
build_path, 'relative-registration')
145+
for entry in os.scandir(relative_registration_dir):
146+
if (entry.is_file() and
147+
re.match(r'tests-[0-9a-f]+_(?:tests|test-list)'
148+
r'(?:-[^.]+)?\.cmake$',
149+
entry.name)):
150+
os.remove(entry.path)
120151

121152
def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
122153
ctest_response = json.loads(ctest_output)
@@ -249,16 +280,9 @@ def escape_catch2_test_names(infos: List[TestInfo]):
249280
escaped.append(TestInfo(name, info.tags))
250281
return escaped
251282

252-
253-
if __name__ == '__main__':
254-
if len(sys.argv) != 3:
255-
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
256-
exit(2)
257-
catch2_path = sys.argv[1]
258-
output_base_path = sys.argv[2]
259-
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
260-
261-
build_path = build_project(sources_dir, output_base_path, catch2_path)
283+
def verify_registration(build_path, test_script_dir=None):
284+
if test_script_dir is None:
285+
test_script_dir = build_path
262286

263287
raw_catch_test_names = get_test_names(build_path)
264288
catch_test_names = escape_catch2_test_names(raw_catch_test_names)
@@ -280,7 +304,8 @@ def escape_catch2_test_names(infos: List[TestInfo]):
280304
exit(1)
281305
print(f"{len(catch_test_names)} tests matched in CTest listing")
282306

283-
test_list_names = sorted(extract_tests_list_from_ctest_script(build_path))
307+
test_list_names = sorted(
308+
extract_tests_list_from_ctest_script(test_script_dir))
284309
expected_names = sorted(info.name for info in raw_catch_test_names)
285310
if test_list_names != expected_names:
286311
print("TEST_LIST variable (tests_TESTS) does not match Catch2 test listing!")
@@ -296,3 +321,34 @@ def escape_catch2_test_names(infos: List[TestInfo]):
296321
cmake_version = get_cmake_version()
297322
if cmake_version >= (3, 27):
298323
check_DL_PATHS(ctest_output)
324+
325+
if __name__ == '__main__':
326+
if len(sys.argv) != 3:
327+
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
328+
exit(2)
329+
catch2_path = sys.argv[1]
330+
output_base_path = sys.argv[2]
331+
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
332+
333+
build_path = build_project(sources_dir, output_base_path, catch2_path)
334+
verify_registration(build_path)
335+
336+
if get_cmake_version() >= (3, 24):
337+
build_project(sources_dir, output_base_path, catch2_path,
338+
use_relative_paths=True)
339+
remove_discovery_cache(build_path)
340+
with tempfile.TemporaryDirectory(
341+
prefix='ctest-registration-relocated-',
342+
dir=output_base_path) as relocation_dir:
343+
relocated_build_path = os.path.join(relocation_dir,
344+
'ctest-registration-test')
345+
os.rename(build_path, relocated_build_path)
346+
try:
347+
verify_registration(
348+
relocated_build_path,
349+
os.path.join(relocated_build_path,
350+
'relative-registration'))
351+
run_ctest(relocated_build_path)
352+
finally:
353+
os.rename(relocated_build_path, build_path)
354+
remove_discovery_cache(build_path)

‎tests/TestScripts/DiscoverTests/relative-registration/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
catch_discover_tests(
2+
tests
3+
ADD_TAGS_AS_LABELS
4+
DISCOVERY_MODE PRE_TEST
5+
USE_RELATIVE_PATHS
6+
WORKING_DIRECTORY "$<TARGET_FILE_DIR:tests>"
7+
${extra_args}
8+
)

0 commit comments

Comments
 (0)