Vulkan: Make the Vulkan renderer thread safe.

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -1297,7 +1297,7 @@ angle::Result RendererVk::getDescriptorSetLayout(
     const vk::DescriptorSetLayoutDesc &desc,
     vk::BindingPointer<vk::DescriptorSetLayout> *descriptorSetLayoutOut)
 {
-    // TODO(geofflang): Synchronize access to the descriptor set layout cache
+    std::lock_guard<decltype(mDescriptorSetLayoutCacheMutex)> lock(mDescriptorSetLayoutCacheMutex);
     return mDescriptorSetLayoutCache.getDescriptorSetLayout(context, desc, descriptorSetLayoutOut);
 }
 
@@ -1307,7 +1307,7 @@ angle::Result RendererVk::getPipelineLayout(
     const vk::DescriptorSetLayoutPointerArray &descriptorSetLayouts,
     vk::BindingPointer<vk::PipelineLayout> *pipelineLayoutOut)
 {
-    // TODO(geofflang): Synchronize access to the pipeline layout cache
+    std::lock_guard<decltype(mPipelineLayoutCacheMutex)> lock(mPipelineLayoutCacheMutex);
     return mPipelineLayoutCache.getPipelineLayout(context, desc, descriptorSetLayouts,
                                                   pipelineLayoutOut);
 }
@@ -1391,23 +1391,31 @@ angle::Result RendererVk::queueSubmit(vk::Context *context,
                                       const VkSubmitInfo &submitInfo,
                                       const vk::Fence &fence)
 {
-    // TODO: synchronize queue access
-    ANGLE_VK_TRY(context, vkQueueSubmit(mQueue, 1, &submitInfo, fence.getHandle()));
+    {
+        std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
+        ANGLE_VK_TRY(context, vkQueueSubmit(mQueue, 1, &submitInfo, fence.getHandle()));
+    }
+
     ANGLE_TRY(cleanupGarbage(context, false));
+
     return angle::Result::Continue;
 }
 
 angle::Result RendererVk::queueWaitIdle(vk::Context *context)
 {
-    // TODO: synchronize queue access
-    ANGLE_VK_TRY(context, vkQueueWaitIdle(mQueue));
+    {
+        std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
+        ANGLE_VK_TRY(context, vkQueueWaitIdle(mQueue));
+    }
+
     ANGLE_TRY(cleanupGarbage(context, false));
+
     return angle::Result::Continue;
 }
 
 VkResult RendererVk::queuePresent(const VkPresentInfoKHR &presentInfo)
 {
-    // TODO: synchronize queue access
+    std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
     return vkQueuePresentKHR(mQueue, &presentInfo);
 }
 
@@ -1427,6 +1435,7 @@ void RendererVk::addGarbage(vk::Shared<vk::Fence> &&fence,
 void RendererVk::addGarbage(std::vector<vk::Shared<vk::Fence>> &&fences,
                             std::vector<vk::GarbageObjectBase> &&garbage)
 {
+    std::lock_guard<decltype(mGarbageMutex)> lock(mGarbageMutex);
     mFencedGarbage.emplace_back(std::move(fences), std::move(garbage));
 }
 
@@ -1468,6 +1477,8 @@ bool RendererVk::hasFormatFeatureBits(VkFormat format, const VkFormatFeatureFlag
 
 angle::Result RendererVk::cleanupGarbage(vk::Context *context, bool block)
 {
+    std::lock_guard<decltype(mGarbageMutex)> lock(mGarbageMutex);
+
     auto garbageIter = mFencedGarbage.begin();
     while (garbageIter != mFencedGarbage.end())
     {

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -12,6 +12,7 @@
 
 #include <vulkan/vulkan.h>
 #include <memory>
+#include <mutex>
 
 #include "common/PoolAlloc.h"
 #include "common/angleutils.h"
@@ -70,7 +71,6 @@ class RendererVk : angle::NonCopyable
     {
         return mPhysicalDeviceFeatures;
     }
-    VkQueue getQueue() const { return mQueue; }
     VkDevice getDevice() const { return mDevice; }
 
     angle::Result selectPresentQueueForSurface(DisplayVk *displayVk,
@@ -183,6 +183,7 @@ class RendererVk : angle::NonCopyable
     VkPhysicalDeviceProperties mPhysicalDeviceProperties;
     VkPhysicalDeviceFeatures mPhysicalDeviceFeatures;
     std::vector<VkQueueFamilyProperties> mQueueFamilyProperties;
+    std::mutex mQueueMutex;
     VkQueue mQueue;
     uint32_t mCurrentQueueFamilyIndex;
     uint32_t mMaxVertexAttribDivisor;
@@ -193,6 +194,7 @@ class RendererVk : angle::NonCopyable
 
     bool mDeviceLost;
 
+    std::mutex mGarbageMutex;
     using FencedGarbage =
         std::pair<std::vector<vk::Shared<vk::Fence>>, std::vector<vk::GarbageObjectBase>>;
     std::vector<FencedGarbage> mFencedGarbage;
@@ -200,6 +202,8 @@ class RendererVk : angle::NonCopyable
     vk::MemoryProperties mMemoryProperties;
     vk::FormatTable mFormatTable;
 
+    // All access to the pipeline cache is done through EGL objects so it is thread safe to not use
+    // a lock.
     vk::PipelineCache mPipelineCache;
     egl::BlobCache::Key mPipelineCacheVkBlobKey;
     uint32_t mPipelineCacheVkUpdateTimeout;
@@ -208,9 +212,11 @@ class RendererVk : angle::NonCopyable
     std::array<VkFormatProperties, vk::kNumVkFormats> mFormatProperties;
 
     // ANGLE uses a PipelineLayout cache to store compatible pipeline layouts.
+    std::mutex mPipelineLayoutCacheMutex;
     PipelineLayoutCache mPipelineLayoutCache;
 
     // DescriptorSetLayouts are also managed in a cache.
+    std::mutex mDescriptorSetLayoutCacheMutex;
     DescriptorSetLayoutCache mDescriptorSetLayoutCache;
 };
 

src/tests/gl_tests/MultithreadingTest.cpp

@@ -29,7 +29,68 @@ class MultithreadingTest : public ANGLETest
         setConfigAlphaBits(8);
     }
 
-    bool platformSupportsMultithreading() const { return (IsOpenGLES() && IsAndroid()); }
+    bool platformSupportsMultithreading() const
+    {
+        return (IsOpenGLES() && IsAndroid()) || IsVulkan();
+    }
+
+    void runMultithreadedGLTest(
+        std::function<void(EGLSurface surface, size_t threadIndex)> testBody,
+        size_t threadCount)
+    {
+        std::mutex mutex;
+
+        EGLWindow *window = getEGLWindow();
+        EGLDisplay dpy    = window->getDisplay();
+        EGLConfig config  = window->getConfig();
+
+        constexpr EGLint kPBufferSize = 256;
+
+        std::vector<std::thread> threads(threadCount);
+        for (size_t threadIdx = 0; threadIdx < threadCount; threadIdx++)
+        {
+            threads[threadIdx] = std::thread([&, threadIdx]() {
+                EGLSurface surface = EGL_NO_SURFACE;
+                EGLConfig ctx      = EGL_NO_CONTEXT;
+
+                {
+                    std::lock_guard<decltype(mutex)> lock(mutex);
+
+                    // Initialize the pbuffer and context
+                    EGLint pbufferAttributes[] = {
+                        EGL_WIDTH, kPBufferSize, EGL_HEIGHT, kPBufferSize, EGL_NONE, EGL_NONE,
+                    };
+                    surface = eglCreatePbufferSurface(dpy, config, pbufferAttributes);
+                    EXPECT_EGL_SUCCESS();
+
+                    ctx = window->createContext(EGL_NO_CONTEXT);
+                    EXPECT_NE(EGL_NO_CONTEXT, ctx);
+
+                    EXPECT_EGL_TRUE(eglMakeCurrent(dpy, surface, surface, ctx));
+                    EXPECT_EGL_SUCCESS();
+                }
+
+                testBody(surface, threadIdx);
+
+                {
+                    std::lock_guard<decltype(mutex)> lock(mutex);
+
+                    // Clean up
+                    EXPECT_EGL_TRUE(
+                        eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT));
+                    EXPECT_EGL_SUCCESS();
+
+                    eglDestroySurface(dpy, surface);
+                    eglDestroyContext(dpy, ctx);
+                }
+            });
+        }
+
+        for (std::thread &thread : threads)
+        {
+            thread.join();
+        }
+    }
 };
 
 // Test that it's possible to make one context current on different threads
@@ -74,84 +135,75 @@ TEST_P(MultithreadingTest, MakeCurrentSingleContext)
     EXPECT_EGL_SUCCESS();
 }
 
-// Test that it's possible to make one multiple contexts current on different threads simultaneously
-TEST_P(MultithreadingTest, MakeCurrentMultiContext)
+// Test that multiple threads can clear and readback pixels successfully at the same time
+TEST_P(MultithreadingTest, MultiContextClear)
 {
     ANGLE_SKIP_TEST_IF(!platformSupportsMultithreading());
-    ANGLE_SKIP_TEST_IF(IsWindows() && IsOpenGL() && IsAMD());
-
-    std::mutex mutex;
 
-    EGLWindow *window = getEGLWindow();
-    EGLDisplay dpy    = window->getDisplay();
-    EGLConfig config  = window->getConfig();
-
-    constexpr size_t kThreadCount         = 16;
-    constexpr size_t kIterationsPerThread = 16;
-
-    constexpr EGLint kPBufferSize = 256;
-
-    std::array<std::thread, kThreadCount> threads;
-    for (size_t thread = 0; thread < kThreadCount; thread++)
-    {
-        threads[thread] = std::thread([&, thread]() {
-            EGLSurface pbuffer = EGL_NO_SURFACE;
-            EGLConfig ctx      = EGL_NO_CONTEXT;
-
-            {
-                std::lock_guard<decltype(mutex)> lock(mutex);
+    auto testBody = [](EGLSurface surface, size_t thread) {
+        constexpr size_t kIterationsPerThread = 32;
+        for (size_t iteration = 0; iteration < kIterationsPerThread; iteration++)
+        {
+            // Base the clear color on the thread and iteration indexes so every clear color is
+            // unique
+            const GLColor color(static_cast<GLubyte>(thread % 255),
+                                static_cast<GLubyte>(iteration % 255), 0, 255);
+            const angle::Vector4 floatColor = color.toNormalizedVector();
+
+            glClearColor(floatColor[0], floatColor[1], floatColor[2], floatColor[3]);
+            EXPECT_GL_NO_ERROR();
+
+            glClear(GL_COLOR_BUFFER_BIT);
+            EXPECT_GL_NO_ERROR();
+
+            EXPECT_PIXEL_COLOR_EQ(0, 0, color);
+        }
+    };
+    runMultithreadedGLTest(testBody, 72);
+}
 
-                // Initialize the pbuffer and context
-                EGLint pbufferAttributes[] = {
-                    EGL_WIDTH, kPBufferSize, EGL_HEIGHT, kPBufferSize, EGL_NONE, EGL_NONE,
-                };
-                pbuffer = eglCreatePbufferSurface(dpy, config, pbufferAttributes);
-                EXPECT_EGL_SUCCESS();
+// Test that multiple threads can draw and readback pixels successfully at the same time
+TEST_P(MultithreadingTest, MultiContextDraw)
+{
+    ANGLE_SKIP_TEST_IF(!platformSupportsMultithreading());
 
-                ctx = window->createContext(EGL_NO_CONTEXT);
-                EXPECT_NE(EGL_NO_CONTEXT, ctx);
+    auto testBody = [](EGLSurface surface, size_t thread) {
+        constexpr size_t kIterationsPerThread = 32;
+        constexpr size_t kDrawsPerIteration   = 500;
 
-                EXPECT_EGL_TRUE(eglMakeCurrent(dpy, pbuffer, pbuffer, ctx));
-                EXPECT_EGL_SUCCESS();
-            }
+        ANGLE_GL_PROGRAM(program, essl1_shaders::vs::Simple(), essl1_shaders::fs::UniformColor());
+        glUseProgram(program);
 
-            for (size_t iteration = 0; iteration < kIterationsPerThread; iteration++)
-            {
-                std::lock_guard<decltype(mutex)> lock(mutex);
+        GLint colorLocation = glGetUniformLocation(program, essl1_shaders::ColorUniform());
 
-                // Base the clear color on the thread and iteration indexes so every clear color is
-                // unique
-                const GLColor color(static_cast<GLubyte>(thread), static_cast<GLubyte>(iteration),
-                                    0, 255);
-                const angle::Vector4 floatColor = color.toNormalizedVector();
+        auto quadVertices = GetQuadVertices();
 
-                glClearColor(floatColor[0], floatColor[1], floatColor[2], floatColor[3]);
-                EXPECT_GL_NO_ERROR();
+        GLBuffer vertexBuffer;
+        glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer);
+        glBufferData(GL_ARRAY_BUFFER, sizeof(GLfloat) * 3 * 6, quadVertices.data(), GL_STATIC_DRAW);
 
-                glClear(GL_COLOR_BUFFER_BIT);
-                EXPECT_GL_NO_ERROR();
+        GLint positionLocation = glGetAttribLocation(program, essl1_shaders::PositionAttrib());
+        glEnableVertexAttribArray(positionLocation);
+        glVertexAttribPointer(positionLocation, 3, GL_FLOAT, GL_FALSE, 0, 0);
 
-                EXPECT_PIXEL_COLOR_EQ(0, 0, color);
-            }
+        for (size_t iteration = 0; iteration < kIterationsPerThread; iteration++)
+        {
+            // Base the clear color on the thread and iteration indexes so every clear color is
+            // unique
+            const GLColor color(static_cast<GLubyte>(thread % 255),
+                                static_cast<GLubyte>(iteration % 255), 0, 255);
+            const angle::Vector4 floatColor = color.toNormalizedVector();
+            glUniform4fv(colorLocation, 1, floatColor.data());
 
+            for (size_t draw = 0; draw < kDrawsPerIteration; draw++)
             {
-                std::lock_guard<decltype(mutex)> lock(mutex);
-
-                // Clean up
-                EXPECT_EGL_TRUE(
-                    eglMakeCurrent(dpy, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT));
-                EXPECT_EGL_SUCCESS();
-
-                eglDestroySurface(dpy, pbuffer);
-                eglDestroyContext(dpy, ctx);
+                glDrawArrays(GL_TRIANGLES, 0, 6);
             }
-        });
-    }
 
-    for (std::thread &thread : threads)
-    {
-        thread.join();
-    }
+            EXPECT_PIXEL_COLOR_EQ(0, 0, color);
+        }
+    };
+    runMultithreadedGLTest(testBody, 4);
 }
 
 // TODO(geofflang): Test sharing a program between multiple shared contexts on multiple threads