VulkanBenchmarks: add validation layers and load via ICD

tests/VulkanBenchmarks/ClearImageBenchmarks.cpp

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "Util.hpp"
 #include "VulkanTester.hpp"
 #include "benchmark/benchmark.h"
 
 #include <cassert>
-
 class ClearImageBenchmark
 {
 public:
@@ -24,6 +24,7 @@ public:
 	{
 		tester.initialize();
 		auto &device = tester.getDevice();
+		auto &physicalDevice = tester.getPhysicalDevice();
 
 		vk::ImageCreateInfo imageInfo;
 		imageInfo.imageType = vk::ImageType::e2D;
@@ -42,7 +43,7 @@ public:
 
 		vk::MemoryAllocateInfo allocateInfo;
 		allocateInfo.allocationSize = memoryRequirements.size;
-		allocateInfo.memoryTypeIndex = 0;
+		allocateInfo.memoryTypeIndex = Util::getMemoryTypeIndex(physicalDevice, memoryRequirements.memoryTypeBits);
 
 		memory = device.allocateMemory(allocateInfo);
 

tests/VulkanBenchmarks/TriangleBenchmarks.cpp

@@ -198,7 +198,7 @@ static void TriangleSampleTexture(benchmark::State &state, Multisample multisamp
 
 			layout(location = 0) out vec4 outColor;
 
-			layout(binding = 0) uniform sampler2D texSampler;
+			layout(binding = 1) uniform sampler2D texSampler;
 
 			void main()
 			{
@@ -221,9 +221,10 @@ static void TriangleSampleTexture(benchmark::State &state, Multisample multisamp
 
 	tester.onUpdateDescriptorSet([](DrawTester &tester, vk::CommandPool &commandPool, vk::DescriptorSet &descriptorSet) {
 		auto &device = tester.getDevice();
+		auto &physicalDevice = tester.getPhysicalDevice();
 		auto &queue = tester.getQueue();
 
-		auto &texture = tester.addImage(device, 16, 16, vk::Format::eR8G8B8A8Unorm).obj;
+		auto &texture = tester.addImage(device, physicalDevice, 16, 16, vk::Format::eR8G8B8A8Unorm).obj;
 
 		// Fill texture with white
 		vk::DeviceSize bufferSize = 16 * 16 * 4;

tests/VulkanWrapper/DrawTester.cpp

@@ -193,7 +193,7 @@ void DrawTester::createFramebuffers(vk::RenderPass renderPass)
 
 	for(size_t i = 0; i < framebuffers.size(); i++)
 	{
-		framebuffers[i].reset(new Framebuffer(device, swapchain->getImageView(i), swapchain->colorFormat, renderPass, swapchain->getExtent(), multisample));
+		framebuffers[i].reset(new Framebuffer(device, physicalDevice, swapchain->getImageView(i), swapchain->colorFormat, renderPass, swapchain->getExtent(), multisample));
 	}
 }
 
@@ -392,10 +392,13 @@ void DrawTester::createCommandBuffers(vk::RenderPass renderPass)
 		}
 
 		// Draw
-		commandBuffers[i].bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline);
-		VULKAN_HPP_NAMESPACE::DeviceSize offset = 0;
-		commandBuffers[i].bindVertexBuffers(0, 1, &vertices.buffer, &offset);
-		commandBuffers[i].draw(vertices.numVertices, 1, 0, 0);
+		if(vertices.numVertices > 0)
+		{
+			commandBuffers[i].bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline);
+			VULKAN_HPP_NAMESPACE::DeviceSize offset = 0;
+			commandBuffers[i].bindVertexBuffers(0, 1, &vertices.buffer, &offset);
+			commandBuffers[i].draw(vertices.numVertices, 1, 0, 0);
+		}
 
 		commandBuffers[i].endRenderPass();
 		commandBuffers[i].end();

tests/VulkanWrapper/DrawTester.hpp

@@ -148,7 +148,7 @@ private:
 		std::vector<vk::VertexInputAttributeDescription> inputAttributes;
 		vk::PipelineVertexInputStateCreateInfo inputState;
 
-		uint32_t numVertices;
+		uint32_t numVertices = 0;
 	} vertices;
 
 	vk::DescriptorSetLayout descriptorSetLayout;  // Owning handle

tests/VulkanWrapper/Framebuffer.cpp

@@ -14,14 +14,14 @@
 
 #include "Framebuffer.hpp"
 
-Framebuffer::Framebuffer(vk::Device device, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample)
+Framebuffer::Framebuffer(vk::Device device, vk::PhysicalDevice physicalDevice, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample)
     : device(device)
 {
 	std::vector<vk::ImageView> attachments(multisample ? 2 : 1);
 
 	if(multisample)
 	{
-		multisampleImage.reset(new Image(device, extent.width, extent.height, colorFormat, vk::SampleCountFlagBits::e4));
+		multisampleImage.reset(new Image(device, physicalDevice, extent.width, extent.height, colorFormat, vk::SampleCountFlagBits::e4));
 
 		// We'll be rendering to attachment location 0
 		attachments[0] = multisampleImage->getImageView();

tests/VulkanWrapper/Framebuffer.hpp

@@ -21,7 +21,7 @@
 class Framebuffer
 {
 public:
-	Framebuffer(vk::Device device, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample);
+	Framebuffer(vk::Device device, vk::PhysicalDevice physicalDevice, vk::ImageView attachment, vk::Format colorFormat, vk::RenderPass renderPass, vk::Extent2D extent, bool multisample);
 	~Framebuffer();
 
 	vk::Framebuffer getFramebuffer()

tests/VulkanWrapper/Image.cpp

@@ -13,8 +13,9 @@
 // limitations under the License.
 
 #include "Image.hpp"
+#include "Util.hpp"
 
-Image::Image(vk::Device device, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount /*= vk::SampleCountFlagBits::e1*/)
+Image::Image(vk::Device device, vk::PhysicalDevice physicalDevice, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount /*= vk::SampleCountFlagBits::e1*/)
     : device(device)
 {
 	vk::ImageCreateInfo imageInfo;
@@ -34,7 +35,7 @@ Image::Image(vk::Device device, uint32_t width, uint32_t height, vk::Format form
 
 	vk::MemoryAllocateInfo allocateInfo;
 	allocateInfo.allocationSize = memoryRequirements.size;
-	allocateInfo.memoryTypeIndex = 0;  //getMemoryTypeIndex(memoryRequirements.memoryTypeBits, vk::MemoryPropertyFlagBits::eDeviceLocal);
+	allocateInfo.memoryTypeIndex = Util::getMemoryTypeIndex(physicalDevice, memoryRequirements.memoryTypeBits);
 
 	imageMemory = device.allocateMemory(allocateInfo);
 

tests/VulkanWrapper/Image.hpp

@@ -20,7 +20,7 @@
 class Image
 {
 public:
-	Image(vk::Device device, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount = vk::SampleCountFlagBits::e1);
+	Image(vk::Device device, vk::PhysicalDevice physicalDevice, uint32_t width, uint32_t height, vk::Format format, vk::SampleCountFlagBits sampleCount = vk::SampleCountFlagBits::e1);
 	~Image();
 
 	vk::Image getImage()

tests/VulkanWrapper/Util.hpp

@@ -22,7 +22,7 @@
 
 namespace Util {
 
-uint32_t getMemoryTypeIndex(vk::PhysicalDevice physicalDevice, uint32_t typeBits, vk::MemoryPropertyFlags properties);
+uint32_t getMemoryTypeIndex(vk::PhysicalDevice physicalDevice, uint32_t typeBits, vk::MemoryPropertyFlags properties = {});
 
 vk::CommandBuffer beginSingleTimeCommands(vk::Device device, vk::CommandPool commandPool);
 

tests/VulkanWrapper/VulkanTester.cpp

@@ -13,7 +13,30 @@
 // limitations under the License.
 
 #include "VulkanTester.hpp"
+#include <cstdlib>
+#include <filesystem>
 #include <fstream>
+#include <iostream>
+
+namespace fs = std::filesystem;
+
+// By default, load SwiftShader via loader
+#ifndef LOAD_NATIVE_DRIVER
+#	define LOAD_NATIVE_DRIVER 0
+#endif
+
+#ifndef LOAD_SWIFTSHADER_DIRECTLY
+#	define LOAD_SWIFTSHADER_DIRECTLY 0
+#endif
+
+#if LOAD_NATIVE_DRIVER && LOAD_SWIFTSHADER_DIRECTLY
+#	error Enable only one of LOAD_NATIVE_DRIVER and LOAD_SWIFTSHADER_DIRECTLY
+#endif
+
+// By default, enable validation layers in DEBUG builds
+#if !defined(ENABLE_VALIDATION_LAYERS) && !defined(NDEBUG)
+#	define ENABLE_VALIDATION_LAYERS 1
+#endif
 
 #if defined(_WIN32)
 #	define OS_WINDOWS 1
@@ -33,7 +56,107 @@
 #	error Unimplemented platform
 #endif
 
+// TODO: move to its own header/cpp
+// Wraps a single environment variable, allowing it to be set
+// and automatically restored on destruction.
+class ScopedSetEnvVar
+{
+public:
+	ScopedSetEnvVar(std::string name)
+	    : name(name)
+	{
+		assert(!name.empty());
+	}
+
+	ScopedSetEnvVar(std::string name, std::string value)
+	    : name(name)
+	{
+		set(value);
+	}
+
+	~ScopedSetEnvVar()
+	{
+		restore();
+	}
+
+	void set(std::string value)
+	{
+		restore();
+		if(auto ov = getEnv(name.data()))
+		{
+			oldValue = ov;
+		}
+		putEnv((name + std::string("=") + value).c_str());
+	}
+
+	void restore()
+	{
+		if(!oldValue.empty())
+		{
+			putEnv((name + std::string("=") + oldValue).c_str());
+			oldValue.clear();
+		}
+	}
+
+private:
+	void putEnv(const char *env)
+	{
+		// POSIX putenv needs 'env' to live beyond the call
+		envCopy = env;
+#if OS_WINDOWS
+		[[maybe_unused]] auto r = ::_putenv(envCopy.c_str());
+		assert(r == 0);
+#else
+		[[maybe_unused]] auto r = ::putenv(const_cast<char *>(envCopy.c_str()));
+		assert(r == 0);
+#endif
+	}
+
+	const char *getEnv(const char *name)
+	{
+		return ::getenv(name);
+	}
+
+	std::string name;
+	std::string oldValue;
+	std::string envCopy;
+};
+
+// Generates a temporary icd.json file that sets library_path at the input driverPath,
+// and sets VK_ICD_FILENAMES environment variable to this file, restoring the env var
+// and deleting the temp file on destruction.
+class ScopedSetIcdFilenames
+{
+public:
+	ScopedSetIcdFilenames() = default;
+	ScopedSetIcdFilenames(const char *driverPath)
+	{
+		std::ofstream fout(icdFileName);
+		assert(fout && "Failed to create generated icd file");
+		fout << R"raw({ "file_format_version": "1.0.0", "ICD": { "library_path": ")raw" << driverPath << R"raw(", "api_version": "1.0.5" } } )raw";
+		fout.close();
+
+		setEnvVar.set(icdFileName);
+	}
+
+	~ScopedSetIcdFilenames()
+	{
+		//TODO(b/180494886): fix C++17 filesystem issues on macOS
+#if !OS_MAC
+		if(fs::exists("vk_swiftshader_generated_icd.json"))
+		{
+			fs::remove("vk_swiftshader_generated_icd.json");
+		}
+#endif
+	}
+
+private:
+	static constexpr const char *icdFileName = "vk_swiftshader_generated_icd.json";
+	ScopedSetEnvVar setEnvVar{ "VK_ICD_FILENAMES" };
+};
+
 namespace {
+
 std::vector<const char *> getDriverPaths()
 {
 #if OS_WINDOWS
@@ -91,24 +214,25 @@ bool fileExists(const char *path)
 	return f.good();
 }
 
-std::unique_ptr<vk::DynamicLoader> loadDriver()
+std::string findDriverPath()
 {
-	for(auto &p : getDriverPaths())
+	for(auto &path : getDriverPaths())
 	{
-		if(!fileExists(p))
-			continue;
-		return std::make_unique<vk::DynamicLoader>(p);
+		if(fileExists(path))
+			return path;
 	}
 
-#if(OS_MAC || OS_LINUX || OS_ANDROID || OS_FUCHSIA)
+#if(OS_LINUX || OS_ANDROID || OS_FUCHSIA)
 	// On Linux-based OSes, the lib path may be resolved by dlopen
-	for(auto &p : getDriverPaths())
+	for(auto &path : getDriverPaths())
 	{
-		auto lib = dlopen(p, RTLD_LAZY | RTLD_LOCAL);
+		auto lib = dlopen(path, RTLD_LAZY | RTLD_LOCAL);
 		if(lib)
 		{
+			char libPath[2048] = { '\0' };
+			dlinfo(lib, RTLD_DI_ORIGIN, libPath);
 			dlclose(lib);
-			return std::make_unique<vk::DynamicLoader>(p);
+			return std::string{ libPath } + "/" + path;
 		}
 	}
 #endif
@@ -118,10 +242,13 @@ std::unique_ptr<vk::DynamicLoader> loadDriver()
 
 }  // namespace
 
+VulkanTester::VulkanTester() = default;
+
 VulkanTester::~VulkanTester()
 {
 	device.waitIdle();
 	device.destroy(nullptr);
+	if(debugReport) instance.destroy(debugReport);
 	instance.destroy(nullptr);
 }
 
@@ -133,9 +260,75 @@ void VulkanTester::initialize()
 	PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr = dl->getProcAddress<PFN_vkGetInstanceProcAddr>("vkGetInstanceProcAddr");
 	VULKAN_HPP_DEFAULT_DISPATCHER.init(vkGetInstanceProcAddr);
 
-	instance = vk::createInstance({}, nullptr);
+	vk::InstanceCreateInfo instanceCreateInfo;
+	std::vector<const char *> extensionNames
+	{
+		VK_KHR_SURFACE_EXTENSION_NAME,
+#if defined(USE_HEADLESS_SURFACE)
+		    VK_EXT_HEADLESS_SURFACE_EXTENSION_NAME,
+#endif
+#if defined(VK_USE_PLATFORM_WIN32_KHR)
+		    VK_KHR_WIN32_SURFACE_EXTENSION_NAME,
+#endif
+	};
+#if ENABLE_VALIDATION_LAYERS
+	extensionNames.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
+#endif
+
+	auto addLayerIfAvailable = [](std::vector<const char *> &layers, const char *layer) {
+		static auto layerProperties = vk::enumerateInstanceLayerProperties();
+		if(std::find_if(layerProperties.begin(), layerProperties.end(), [layer](auto &lp) {
+			   return strcmp(layer, lp.layerName) == 0;
+		   }) != layerProperties.end())
+		{
+			//std::cout << "Enabled layer: " << layer << std::endl;
+			layers.push_back(layer);
+		}
+	};
+
+	std::vector<const char *> layerNames;
+#if ENABLE_VALIDATION_LAYERS
+	addLayerIfAvailable(layerNames, "VK_LAYER_KHRONOS_validation");
+	addLayerIfAvailable(layerNames, "VK_LAYER_LUNARG_standard_validation");
+#endif
+
+	instanceCreateInfo.ppEnabledExtensionNames = extensionNames.data();
+	instanceCreateInfo.enabledExtensionCount = static_cast<uint32_t>(extensionNames.size());
+	instanceCreateInfo.ppEnabledLayerNames = layerNames.data();
+	instanceCreateInfo.enabledLayerCount = static_cast<uint32_t>(layerNames.size());
+
+	instance = vk::createInstance(instanceCreateInfo, nullptr);
 	VULKAN_HPP_DEFAULT_DISPATCHER.init(instance);
 
+#if ENABLE_VALIDATION_LAYERS
+	if(VULKAN_HPP_DEFAULT_DISPATCHER.vkCreateDebugUtilsMessengerEXT)
+	{
+		vk::DebugUtilsMessengerCreateInfoEXT debugInfo;
+		debugInfo.messageSeverity =
+		    // vk::DebugUtilsMessageSeverityFlagBitsEXT::eVerbose |
+		    vk::DebugUtilsMessageSeverityFlagBitsEXT::eError |
+		    vk::DebugUtilsMessageSeverityFlagBitsEXT::eWarning;
+
+		debugInfo.messageType = vk::DebugUtilsMessageTypeFlagBitsEXT::eGeneral |
+		                        vk::DebugUtilsMessageTypeFlagBitsEXT::eValidation |
+		                        vk::DebugUtilsMessageTypeFlagBitsEXT::ePerformance;
+
+		PFN_vkDebugUtilsMessengerCallbackEXT debugInfoCallback =
+		    [](
+		        VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity,
+		        VkDebugUtilsMessageTypeFlagsEXT messageTypes,
+		        const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData,
+		        void *pUserData) -> VkBool32 {
+			//assert(false);
+			std::cerr << "[DebugInfoCallback] " << pCallbackData->pMessage << std::endl;
+			return VK_FALSE;
+		};
+
+		debugInfo.pfnUserCallback = debugInfoCallback;
+		debugReport = instance.createDebugUtilsMessengerEXT(debugInfo);
+	}
+#endif
+
 	std::vector<vk::PhysicalDevice> physicalDevices = instance.enumeratePhysicalDevices();
 	assert(!physicalDevices.empty());
 	physicalDevice = physicalDevices[0];
@@ -146,11 +339,56 @@ void VulkanTester::initialize()
 	queueCreateInfo.queueCount = 1;
 	queueCreateInfo.pQueuePriorities = &defaultQueuePriority;
 
+	std::vector<const char *> deviceExtensions = {
+		VK_KHR_SWAPCHAIN_EXTENSION_NAME,
+	};
+
 	vk::DeviceCreateInfo deviceCreateInfo;
 	deviceCreateInfo.queueCreateInfoCount = 1;
 	deviceCreateInfo.pQueueCreateInfos = &queueCreateInfo;
+	deviceCreateInfo.ppEnabledExtensionNames = deviceExtensions.data();
+	deviceCreateInfo.enabledExtensionCount = static_cast<uint32_t>(deviceExtensions.size());
 
 	device = physicalDevice.createDevice(deviceCreateInfo, nullptr);
 
 	queue = device.getQueue(queueFamilyIndex, 0);
 }
+
+std::unique_ptr<vk::DynamicLoader> VulkanTester::loadDriver()
+{
+	if(LOAD_NATIVE_DRIVER)
+	{
+		return std::make_unique<vk::DynamicLoader>();
+	}
+
+	auto driverPath = findDriverPath();
+	assert(!driverPath.empty());
+
+	if(LOAD_SWIFTSHADER_DIRECTLY)
+	{
+		return std::make_unique<vk::DynamicLoader>(driverPath);
+	}
+
+	// Load SwiftShader via loader
+
+	// Set VK_ICD_FILENAMES env var so it gets picked up by the loading of the ICD driver
+	setIcdFilenames = std::make_unique<ScopedSetIcdFilenames>(driverPath.c_str());
+
+	std::unique_ptr<vk::DynamicLoader> dl;
+#ifndef VULKAN_HPP_NO_EXCEPTIONS
+	try
+	{
+		dl = std::make_unique<vk::DynamicLoader>();
+	}
+	catch(std::exception &ex)
+	{
+		std::cerr << "vk::DynamicLoader exception: " << ex.what() << std::endl;
+		std::cerr << "Falling back to loading SwiftShader directly (i.e. no validation layers)" << std::endl;
+		dl = std::make_unique<vk::DynamicLoader>(driverPath);
+	}
+#else
+	dl = std::make_unique<vk::DynamicLoader>();
+#endif
+
+	return dl;
+}

tests/VulkanWrapper/VulkanTester.hpp

@@ -20,19 +20,24 @@
 class VulkanTester
 {
 public:
-	VulkanTester() = default;
+	VulkanTester();
 	virtual ~VulkanTester();
 
 	// Call once after construction so that virtual functions may be called during init
 	void initialize();
 
 	const vk::DynamicLoader &dynamicLoader() const { return *dl; }
-	vk::Device &getDevice() { return this->device; }
-	vk::Queue &getQueue() { return this->queue; }
+	vk::PhysicalDevice &getPhysicalDevice() { return physicalDevice; }
+	vk::Device &getDevice() { return device; }
+	vk::Queue &getQueue() { return queue; }
 	uint32_t getQueueFamilyIndex() const { return queueFamilyIndex; }
 
 private:
+	std::unique_ptr<vk::DynamicLoader> loadDriver();
+
+	std::unique_ptr<class ScopedSetIcdFilenames> setIcdFilenames;
 	std::unique_ptr<vk::DynamicLoader> dl;
+	vk::DebugUtilsMessengerEXT debugReport;
 
 protected:
 	const uint32_t queueFamilyIndex = 0;

tests/VulkanWrapper/Window.cpp

@@ -17,6 +17,7 @@
 #if USE_HEADLESS_SURFACE
 
 Window::Window(vk::Instance instance, vk::Extent2D windowSize)
+    : instance(instance)
 {
 	vk::HeadlessSurfaceCreateInfoEXT surfaceCreateInfo;
 	surface = instance.createHeadlessSurfaceEXT(surfaceCreateInfo);
@@ -40,6 +41,7 @@ void Window::show()
 #elif defined(_WIN32)
 
 Window::Window(vk::Instance instance, vk::Extent2D windowSize)
+    : instance(instance)
 {
 	windowClass.cbSize = sizeof(WNDCLASSEX);
 	windowClass.style = CS_HREDRAW | CS_VREDRAW;