Skip to content

Commit 77c8a2f

Browse files
NgoQuocViet2001NgoQuocViet2001
committed
Fix relative PRE_TEST discovery paths
Closes #3051
1 parent a15f718 commit 77c8a2f

5 files changed

Lines changed: 173 additions & 38 deletions

File tree

‎docs/cmake-integration.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ catch_discover_tests(target
127127
[OUTPUT_PREFIX prefix]
128128
[OUTPUT_SUFFIX suffix]
129129
[DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
130+
[USE_RELATIVE_PATHS]
130131
[SKIP_IS_FAILURE]
131132
[ADD_TAGS_AS_LABELS]
132133
)
@@ -212,6 +213,12 @@ execution (useful e.g. in cross-compilation environments).
212213
calling ``catch_discover_tests``. This provides a mechanism for globally
213214
selecting a preferred test discovery behavior.
214215

216+
* `USE_RELATIVE_PATHS`
217+
218+
With `DISCOVERY_MODE PRE_TEST`, stores generated discovery paths relative to
219+
the CTest include file. This allows the build tree and test executable to be
220+
relocated before test discovery runs. This option requires CMake 3.24 or newer.
221+
215222
* `SKIP_IS_FAILURE`
216223

217224
Skipped tests will be marked as failed instead.

‎extras/Catch.cmake‎

Lines changed: 79 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
3838
[OUTPUT_PREFIX prefix]
3939
[OUTPUT_SUFFIX suffix]
4040
[DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
41+
[USE_RELATIVE_PATHS]
4142
[SKIP_IS_FAILURE]
4243
[ADD_TAGS_AS_LABELS]
4344
)
@@ -146,6 +147,12 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
146147
calling ``catch_discover_tests``. This provides a mechanism for globally selecting
147148
a preferred test discovery behavior without having to modify each call site.
148149
150+
``USE_RELATIVE_PATHS``
151+
Makes the files generated for ``PRE_TEST`` discovery use paths relative to
152+
the generated CTest include file. This allows a build tree to be relocated
153+
with its test executable before test discovery runs. This option requires
154+
CMake 3.24 or newer.
155+
149156
``SKIP_IS_FAILURE``
150157
Disables skipped test detection.
151158
@@ -159,7 +166,7 @@ function(catch_discover_tests TARGET)
159166

160167
cmake_parse_arguments(
161168
""
162-
"SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS"
169+
"SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS;USE_RELATIVE_PATHS"
163170
"TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE"
164171
"TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS"
165172
${ARGN}
@@ -190,6 +197,12 @@ function(catch_discover_tests TARGET)
190197
if(NOT _DISCOVERY_MODE MATCHES "^(POST_BUILD|PRE_TEST)$")
191198
message(FATAL_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}")
192199
endif()
200+
if(_USE_RELATIVE_PATHS AND NOT _DISCOVERY_MODE STREQUAL "PRE_TEST")
201+
message(FATAL_ERROR "USE_RELATIVE_PATHS requires DISCOVERY_MODE PRE_TEST")
202+
endif()
203+
if(_USE_RELATIVE_PATHS AND CMAKE_VERSION VERSION_LESS "3.24")
204+
message(FATAL_ERROR "USE_RELATIVE_PATHS requires CMake 3.24 or newer")
205+
endif()
193206

194207
## Generate a unique name based on the extra arguments
195208
string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX}")
@@ -199,6 +212,7 @@ function(catch_discover_tests TARGET)
199212
set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-${args_hash}")
200213
set(ctest_include_file "${ctest_file_base}_include.cmake")
201214
set(ctest_tests_file "${ctest_file_base}_tests.cmake")
215+
set(ctest_include_file_for_property "${ctest_include_file}")
202216

203217
get_property(crosscompiling_emulator
204218
TARGET ${TARGET}
@@ -253,16 +267,69 @@ function(catch_discover_tests TARGET)
253267
set(ctest_tests_file "${ctest_file_base}_tests-$<CONFIG>.cmake")
254268
endif()
255269

270+
set(test_executable_for_script "$<TARGET_FILE:${TARGET}>")
271+
set(test_working_dir_for_script "${_WORKING_DIRECTORY}")
272+
set(discover_tests_script_for_script "${_CATCH_DISCOVER_TESTS_SCRIPT}")
273+
set(ctest_tests_file_for_script "${ctest_tests_file}")
274+
set(test_executable_argument "[==[${test_executable_for_script}]==]")
275+
set(test_working_dir_argument "[==[${test_working_dir_for_script}]==]")
276+
set(ctest_file_argument "[==[${ctest_tests_file_for_script}]==]")
277+
set(ctest_path_setup)
278+
set(ctest_config_include_file
279+
"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake"
280+
)
281+
282+
if(_USE_RELATIVE_PATHS)
283+
get_filename_component(ctest_file_base_name "${ctest_file_base}" NAME)
284+
get_filename_component(ctest_tests_file_name "${ctest_tests_file}" NAME)
285+
get_filename_component(ctest_include_file_name "${ctest_include_file}" NAME)
286+
287+
set(test_executable_relative
288+
"$<PATH:RELATIVE_PATH,$<TARGET_FILE:${TARGET}>,${CMAKE_CURRENT_BINARY_DIR}>"
289+
)
290+
set(test_working_dir_relative
291+
"$<PATH:RELATIVE_PATH,$<PATH:ABSOLUTE_PATH,${_WORKING_DIRECTORY},${CMAKE_CURRENT_BINARY_DIR}>,${CMAKE_CURRENT_BINARY_DIR}>"
292+
)
293+
set(discover_tests_script_name
294+
"${ctest_file_base_name}_CatchAddTests.cmake"
295+
)
296+
configure_file(
297+
"${_CATCH_DISCOVER_TESTS_SCRIPT}"
298+
"${CMAKE_CURRENT_BINARY_DIR}/${discover_tests_script_name}"
299+
COPYONLY
300+
)
301+
302+
set(ctest_config_include_file
303+
"${ctest_file_base_name}_include-\${CTEST_CONFIGURATION_TYPE}.cmake"
304+
)
305+
string(CONCAT ctest_path_setup
306+
"get_filename_component(_catch_discover_tests_script \"${discover_tests_script_name}\" REALPATH)" "\n"
307+
"get_filename_component(_catch_discovery_dir \"\${_catch_discover_tests_script}\" DIRECTORY)" "\n"
308+
"set(_catch_test_executable \"\${_catch_discovery_dir}/${test_executable_relative}\")" "\n"
309+
"set(_catch_test_working_dir \"\${_catch_discovery_dir}/${test_working_dir_relative}\")" "\n"
310+
"set(_catch_tests_file \"\${_catch_discovery_dir}/${ctest_tests_file_name}\")" "\n"
311+
)
312+
set(test_executable_for_script "\${_catch_test_executable}")
313+
set(test_working_dir_for_script "\${_catch_test_working_dir}")
314+
set(discover_tests_script_for_script "\${_catch_discover_tests_script}")
315+
set(ctest_tests_file_for_script "\${_catch_tests_file}")
316+
set(test_executable_argument "\"\${_catch_test_executable}\"")
317+
set(test_working_dir_argument "\"\${_catch_test_working_dir}\"")
318+
set(ctest_file_argument "\"\${_catch_tests_file}\"")
319+
set(ctest_include_file_for_property "${ctest_include_file_name}")
320+
endif()
321+
256322
string(CONCAT ctest_include_content
257-
"if(EXISTS \"$<TARGET_FILE:${TARGET}>\")" "\n"
258-
" if(NOT EXISTS \"${ctest_tests_file}\" OR" "\n"
259-
" NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"$<TARGET_FILE:${TARGET}>\" OR\n"
260-
" NOT \"${ctest_tests_file}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n"
261-
" include(\"${_CATCH_DISCOVER_TESTS_SCRIPT}\")" "\n"
323+
"${ctest_path_setup}"
324+
"if(EXISTS \"${test_executable_for_script}\")" "\n"
325+
" if(NOT EXISTS \"${ctest_tests_file_for_script}\" OR" "\n"
326+
" NOT \"${ctest_tests_file_for_script}\" IS_NEWER_THAN \"${test_executable_for_script}\" OR\n"
327+
" NOT \"${ctest_tests_file_for_script}\" IS_NEWER_THAN \"\${CMAKE_CURRENT_LIST_FILE}\")\n"
328+
" include(\"${discover_tests_script_for_script}\")" "\n"
262329
" catch_discover_tests_impl(" "\n"
263-
" TEST_EXECUTABLE" " [==[" "$<TARGET_FILE:${TARGET}>" "]==]" "\n"
330+
" TEST_EXECUTABLE" " ${test_executable_argument}" "\n"
264331
" TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n"
265-
" TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n"
332+
" TEST_WORKING_DIR" " ${test_working_dir_argument}" "\n"
266333
" TEST_SPEC" " [==[" "${_TEST_SPEC}" "]==]" "\n"
267334
" TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n"
268335
" TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n"
@@ -273,13 +340,13 @@ function(catch_discover_tests TARGET)
273340
" TEST_OUTPUT_DIR" " [==[" "${_OUTPUT_DIR}" "]==]" "\n"
274341
" TEST_OUTPUT_PREFIX" " [==[" "${_OUTPUT_PREFIX}" "]==]" "\n"
275342
" TEST_OUTPUT_SUFFIX" " [==[" "${_OUTPUT_SUFFIX}" "]==]" "\n"
276-
" CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n"
343+
" CTEST_FILE" " ${ctest_file_argument}" "\n"
277344
" TEST_DL_PATHS" " [==[" "${_DL_PATHS}" "]==]" "\n"
278345
" TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}" "]==]" "\n"
279346
" ADD_TAGS_AS_LABELS" " [==[" "${_ADD_TAGS_AS_LABELS}" "]==]" "\n"
280347
" )" "\n"
281348
" endif()" "\n"
282-
" include(\"${ctest_tests_file}\")" "\n"
349+
" include(\"${ctest_tests_file_for_script}\")" "\n"
283350
"else()" "\n"
284351
" add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n"
285352
"endif()" "\n"
@@ -293,7 +360,7 @@ function(catch_discover_tests TARGET)
293360
"if(NOT CTEST_CONFIGURATION_TYPE)" "\n"
294361
" message(\"No configuration for testing specified, use '-C <cfg>'.\")" "\n"
295362
"else()" "\n"
296-
" include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")" "\n"
363+
" include(\"${ctest_config_include_file}\")" "\n"
297364
"endif()" "\n"
298365
)
299366
file(GENERATE OUTPUT "${ctest_include_file}" CONTENT "${ctest_include_multi_content}")
@@ -305,7 +372,7 @@ function(catch_discover_tests TARGET)
305372

306373
# Add discovered tests to directory TEST_INCLUDE_FILES
307374
set_property(DIRECTORY
308-
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
375+
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file_for_property}"
309376
)
310377

311378
endfunction()

‎tests/TestScripts/DiscoverTests/CMakeLists.txt‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,15 @@ if(CMAKE_VERSION GREATER_EQUAL 3.27)
1313
DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
1414
)
1515
endif()
16-
catch_discover_tests(
17-
tests
18-
ADD_TAGS_AS_LABELS
19-
DISCOVERY_MODE PRE_TEST
20-
${extra_args}
21-
)
16+
if(CATCH2_TEST_USE_RELATIVE_PATHS)
17+
add_subdirectory(relative-registration)
18+
else()
19+
catch_discover_tests(
20+
tests
21+
ADD_TAGS_AS_LABELS
22+
DISCOVERY_MODE PRE_TEST
23+
${extra_args}
24+
)
25+
endif()
2226

2327
# DISCOVERY_MODE <POST_BUILD|PRE_TEST>

‎tests/TestScripts/DiscoverTests/VerifyRegistration.py‎

Lines changed: 69 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,17 @@ def get_cmake_version():
3434
int(version_match.group(2)),
3535
int(version_match.group(3)))
3636

37-
def build_project(sources_dir, output_base_path, catch2_path):
37+
def build_project(sources_dir, output_base_path, catch2_path,
38+
use_relative_paths=False):
3839
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
3940
config_cmd = ['cmake',
4041
'-B', build_dir,
4142
'-S', sources_dir,
4243
f'-DCATCH2_PATH={catch2_path}',
4344
'-DCMAKE_BUILD_TYPE=Debug']
45+
relative_paths_setting = 'ON' if use_relative_paths else 'OFF'
46+
config_cmd.append(
47+
f'-DCATCH2_TEST_USE_RELATIVE_PATHS={relative_paths_setting}')
4448

4549
build_cmd = ['cmake',
4650
'--build', build_dir,
@@ -99,23 +103,49 @@ def get_test_names(build_path: str) -> List[TestInfo]:
99103

100104
def get_ctest_listing(build_path):
101105
old_path = os.getcwd()
102-
os.chdir(build_path)
106+
try:
107+
os.chdir(build_path)
108+
109+
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
110+
try:
111+
result = subprocess.run(cmd,
112+
capture_output = True,
113+
check = True,
114+
text = True)
115+
except subprocess.CalledProcessError as err:
116+
print('Error when getting output from CTest')
117+
print(f'cmd: {err.cmd}')
118+
print(f'stderr: {err.stderr}')
119+
print(f'stdout: {err.stdout}')
120+
exit(4)
121+
finally:
122+
os.chdir(old_path)
123+
124+
return result.stdout
103125

104-
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
126+
def run_ctest(build_path):
127+
cmd = ['ctest', '-C', 'debug', '--output-on-failure']
105128
try:
106-
result = subprocess.run(cmd,
107-
capture_output = True,
108-
check = True,
109-
text = True)
129+
subprocess.run(cmd,
130+
capture_output = True,
131+
check = True,
132+
cwd = build_path,
133+
text = True)
110134
except subprocess.CalledProcessError as err:
111-
print('Error when getting output from CTest')
135+
print('Error when running discovered tests')
112136
print(f'cmd: {err.cmd}')
113137
print(f'stderr: {err.stderr}')
114138
print(f'stdout: {err.stdout}')
115139
exit(4)
116140

117-
os.chdir(old_path)
118-
return result.stdout
141+
def remove_discovery_cache(build_path):
142+
relative_registration_dir = os.path.join(
143+
build_path, 'relative-registration')
144+
for entry in os.scandir(relative_registration_dir):
145+
if (entry.is_file() and
146+
re.match(r'tests-[0-9a-f]+_tests(?:-[^.]+)?\.cmake$',
147+
entry.name)):
148+
os.remove(entry.path)
119149

120150
def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
121151
ctest_response = json.loads(ctest_output)
@@ -155,16 +185,7 @@ def escape_catch2_test_names(infos: List[TestInfo]):
155185
escaped.append(TestInfo(name, info.tags))
156186
return escaped
157187

158-
if __name__ == '__main__':
159-
if len(sys.argv) != 3:
160-
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
161-
exit(2)
162-
catch2_path = sys.argv[1]
163-
output_base_path = sys.argv[2]
164-
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
165-
166-
build_path = build_project(sources_dir, output_base_path, catch2_path)
167-
188+
def verify_registration(build_path):
168189
catch_test_names = escape_catch2_test_names(get_test_names(build_path))
169190
ctest_output = get_ctest_listing(build_path)
170191
ctest_test_names = extract_tests_from_ctest(ctest_output)
@@ -187,3 +208,31 @@ def escape_catch2_test_names(infos: List[TestInfo]):
187208
cmake_version = get_cmake_version()
188209
if cmake_version >= (3, 27):
189210
check_DL_PATHS(ctest_output)
211+
212+
if __name__ == '__main__':
213+
if len(sys.argv) != 3:
214+
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
215+
exit(2)
216+
catch2_path = sys.argv[1]
217+
output_base_path = sys.argv[2]
218+
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
219+
220+
build_path = build_project(sources_dir, output_base_path, catch2_path)
221+
verify_registration(build_path)
222+
223+
if get_cmake_version() >= (3, 24):
224+
build_project(sources_dir, output_base_path, catch2_path,
225+
use_relative_paths=True)
226+
remove_discovery_cache(build_path)
227+
with tempfile.TemporaryDirectory(
228+
prefix='ctest-registration-relocated-',
229+
dir=output_base_path) as relocation_dir:
230+
relocated_build_path = os.path.join(relocation_dir,
231+
'ctest-registration-test')
232+
os.rename(build_path, relocated_build_path)
233+
try:
234+
verify_registration(relocated_build_path)
235+
run_ctest(relocated_build_path)
236+
finally:
237+
os.rename(relocated_build_path, build_path)
238+
remove_discovery_cache(build_path)

‎tests/TestScripts/DiscoverTests/relative-registration/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
catch_discover_tests(
2+
tests
3+
ADD_TAGS_AS_LABELS
4+
DISCOVERY_MODE PRE_TEST
5+
USE_RELATIVE_PATHS
6+
WORKING_DIRECTORY "$<TARGET_FILE_DIR:tests>"
7+
${extra_args}
8+
)

0 commit comments

Comments
 (0)