Update tracetooltests and improve testing

We now exit replay with exit code 77 if we find a missing
feature or extension to play nice with automated testing.

Author: per-mathisen-arm

CMakeLists.txt

@@ -508,15 +508,15 @@ function(layer_test test_name test_exe)
 	add_test(NAME layer_test_${ARGV0}_replay COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lava-replay -V vulkan_${ARGV0}.vk)
 	set_tests_properties(layer_test_${ARGV0}_replay PROPERTIES FIXTURES_REQUIRED ${ARGV0})
 	add_test(NAME layer_test_${ARGV0}_replay_cpu COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lava-replay -C -V vulkan_${ARGV0}.vk)
-	set_tests_properties(layer_test_${ARGV0}_replay_cpu PROPERTIES FIXTURES_REQUIRED ${ARGV0})
+	set_tests_properties(layer_test_${ARGV0}_replay_cpu PROPERTIES FIXTURES_REQUIRED ${ARGV0} SKIP_RETURN_CODE 77)
 	add_test(NAME layer_test_${ARGV0}_replay_blackhole COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lava-replay -V -B vulkan_${ARGV0}.vk)
 	set_tests_properties(layer_test_${ARGV0}_replay_blackhole PROPERTIES FIXTURES_REQUIRED ${ARGV0})
 	add_test(NAME layer_test_${ARGV0}_replay_blackhole_nonewsi COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lava-replay -V -B -w none vulkan_${ARGV0}.vk)
 	set_tests_properties(layer_test_${ARGV0}_replay_blackhole_nonewsi PROPERTIES FIXTURES_REQUIRED ${ARGV0})
 	add_test(NAME layer_test_${ARGV0}_replay_nonewsi COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lava-replay -V -w none vulkan_${ARGV0}.vk)
 	set_tests_properties(layer_test_${ARGV0}_replay_nonewsi PROPERTIES FIXTURES_REQUIRED ${ARGV0})
 	add_test(NAME layer_test_${ARGV0}_replay_from_simulation COMMAND ${CMAKE_CURRENT_BINARY_DIR}/lava-replay -V vulkan_${ARGV0}_simulation.vk)
-	set_tests_properties(layer_test_${ARGV0}_replay_from_simulation PROPERTIES FIXTURES_REQUIRED "${ARGV0}_simulation;${ARGV0}")
+	set_tests_properties(layer_test_${ARGV0}_replay_from_simulation PROPERTIES FIXTURES_REQUIRED "${ARGV0}_simulation;${ARGV0}" SKIP_RETURN_CODE 77)
 endfunction()
 
 layer_test(general general)
@@ -553,6 +553,7 @@ layer_test(thread_4_2 thread_4 -V 3 -S 100)
 layer_test(thread_4_3 thread_4 -V 2 -S 0)
 layer_test(compute_bda_sc compute_bda_sc)
 layer_test(compute_bda_pushconstant compute_bda_pushconstant)
+layer_test(compute_bda_pushconstant_4 compute_bda_pushconstant -V 4)
 layer_test(compute_bda_ubo compute_bda_ubo)
 layer_test(compute_bda_ubo_ssbo compute_bda_ubo --ssbo)
 layer_test(compute_bda_ubo_fb compute_bda_ubo --frame-boundary)
@@ -618,6 +619,7 @@ layer_test(pipeline_executable_properties pipeline_executable_properties)
 layer_test(extended_dynamic_state3 extended_dynamic_state3)
 #layer_test(push_descriptor)
 layer_test(host_image_copy host_image_copy)
+layer_test(host_image_copy_ext host_image_copy_ext)
 #layer_test(compute_aliasing compute_aliasing)
 layer_test(compute_bda_copying_address compute_bda_copying_address)
 layer_test(compute_descriptor_buffer compute_descriptor_buffer)
@@ -631,3 +633,7 @@ layer_test(tensors_1 tensors_1)
 layer_test(timeline_semaphore_1 timeline_semaphore_1)
 layer_test(trace_helpers trace_helpers)
 layer_test(updatedescriptor_1 updatedescriptor_1)
+layer_test(device_generated_commands_1 device_generated_commands_1)
+layer_test(nested_commandbuffers nested_commandbuffers)
+#layer_test(compute_shader_module_identifier compute_shader_module_identifier)
+layer_test(compute_shader_object compute_shader_object)

external/tracetooltests

@@ -1469,6 +1469,11 @@ def loadfunc(name, node, target, header):
 		z.do('if (reader.run)')
 		z.brace_begin()
 		z.do('retval = vkuSetupDevice(%s);' % (', '.join(call_list)))
+		z.do('if (retval == VK_ERROR_FEATURE_NOT_PRESENT || retval == VK_ERROR_EXTENSION_NOT_PRESENT)')
+		z.brace_begin()
+		z.do('reader.parent->exit_status = 77;')
+		z.do('reader.parent->finalize(true);') # this actually exits now, since we cancel our own thread
+		z.brace_end()
 		z.do('check_retval(stored_retval, retval);')
 		z.brace_end()
 	elif name == "vkGetFenceStatus": # wait for success to restore original synchronization when call was originally successful

scripts/util.py

@@ -214,6 +214,7 @@ def print_wrapper_cpp(name):
 	print('\tVkResult retval = wrap_vkCreateDevice(physicalDevice, pCreateInfo, pAllocator, pDevice);', file=w)
 	print('\tif (retval == VK_ERROR_FEATURE_NOT_PRESENT) print_feature_mismatch(physicalDevice, pCreateInfo);', file=w)
 	print('\telse if (retval == VK_ERROR_EXTENSION_NOT_PRESENT) print_extension_mismatch(physicalDevice, pCreateInfo);', file=w)
+	print('\tif (retval == VK_ERROR_FEATURE_NOT_PRESENT || retval == VK_ERROR_EXTENSION_NOT_PRESENT) { return retval; }', file=w) # handle properly later
 	print('\tif (retval != VK_SUCCESS) ABORT("Failed to create device: %s", errorString(retval));', file=w)
 	print('\tassert(*pDevice);', file=w)
 	print('\tPFN_vkGetDeviceProcAddr gdpa = wrap_vkGetDeviceProcAddr;', file=w)

scripts/vk_wrapper.py

@@ -31,7 +31,9 @@ static trackable* debug_object_trackable(trace_records& r, VkDebugReportObjectTy
 	switch (type)
 	{
 	case VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT: return r.VkInstance_index.at((const VkInstance)object);
-	case VK_DEBUG_REPORT_OBJECT_TYPE_PHYSICAL_DEVICE_EXT: return r.VkPhysicalDevice_index.at((VkPhysicalDevice)object);
+	case VK_DEBUG_REPORT_OBJECT_TYPE_PHYSICAL_DEVICE_EXT:
+		if (r.VkPhysicalDevice_index.contains((VkPhysicalDevice)object)) return r.VkPhysicalDevice_index.at((VkPhysicalDevice)object);
+		else return nullptr;
 	case VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT: return r.VkDevice_index.at((VkDevice)object);
 	case VK_DEBUG_REPORT_OBJECT_TYPE_QUEUE_EXT: return (p__virtualqueues) ? (trackable*)object : r.VkQueue_index.at((const VkQueue)object);
 	case VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_MEMORY_EXT: return r.VkDeviceMemory_index.at((const VkDeviceMemory)object);

src/hardcode_write.cpp

@@ -139,6 +139,7 @@ void lava_reader::finalize(bool terminate)
 	out["process_time"] = process_time;
 	if (terminate)
 	{
+		if (p__debug_destination) fflush(p__debug_destination);
 		for (auto& v : *thread_call_numbers) v = 0; // stop waiting threads from progressing
 		for (unsigned i = 0; i < threads.size(); i++)
 		{

src/read.cpp

@@ -4,7 +4,6 @@
 
 #include <assert.h>
 #include <atomic>
-#include <thread>
 #include <cstdint>
 #include <thread>
 #include <list>
@@ -97,6 +96,9 @@ class lava_reader
 	int stored_version_minor = 0;
 	int stored_version_patch = 0;
 
+	/// Set our exit status here if we want to signal something special (eg a skip test value)
+	std::atomic_int exit_status { 0 };
+
 private:
 	/// Start time of frame range
 	std::atomic_uint64_t mStartTime{ 0 };

src/read.h

@@ -307,5 +307,5 @@ int main(int argc, char **argv)
 	wsi_shutdown();
 	vkuDestroyWrapper(library);
 	if (p__debug_destination) fclose(p__debug_destination);
-	return 0;
+	return replayer.exit_status;
 }

src/replay.cpp

@@ -154,5 +154,4 @@ void print_feature_mismatch(VkPhysicalDevice physicalDevice, const VkDeviceCreat
 	if (pEnabledFeatures->inheritedQueries == true && device_features.inheritedQueries == false) ILOG("\tinheritedQueries");
 	// Check pNext feature structs
 	print_pnext_feature_mismatches(physicalDevice, pCreateInfo);
-	ABORT("Mismatched feature sets!");
 }