Vulkan: Move device queue management to CommandQueue.

src/libANGLE/renderer/vulkan/CommandProcessor.cpp

@@ -148,7 +148,8 @@ void CommandProcessorTask::copyPresentInfo(const VkPresentInfoKHR &other)
     }
 }
 
-void CommandProcessorTask::initPresent(egl::ContextPriority priority, VkPresentInfoKHR &presentInfo)
+void CommandProcessorTask::initPresent(egl::ContextPriority priority,
+                                       const VkPresentInfoKHR &presentInfo)
 {
     mTask     = CustomTask::Present;
     mPriority = priority;
@@ -303,8 +304,15 @@ void CommandProcessor::queueCommand(Context *context, CommandProcessorTask *task
     mWorkAvailableCondition.notify_one();
 }
 
-void CommandProcessor::processTasks()
+void CommandProcessor::processTasks(const DeviceQueueMap &queueMap)
 {
+    angle::Result initResult = mCommandQueue.init(this, queueMap);
+    if (initResult == angle::Result::Stop)
+    {
+        // TODO: https://issuetracker.google.com/issues/170311829 - error handling
+        UNREACHABLE();
+        return;
+    }
 
     while (true)
     {
@@ -328,8 +336,6 @@ void CommandProcessor::processTasks()
 
 angle::Result CommandProcessor::processTasksImpl(bool *exitThread)
 {
-    ANGLE_TRY(mCommandQueue.init(this));
-
     while (true)
     {
         std::unique_lock<std::mutex> lock(mWorkerMutex);
@@ -420,8 +426,7 @@ angle::Result CommandProcessor::processTask(CommandProcessorTask *task)
         }
         case CustomTask::Present:
         {
-            VkResult result =
-                present(mRenderer->getVkQueue(task->getPriority()), task->getPresentInfo());
+            VkResult result = present(task->getPriority(), task->getPresentInfo());
             if (ANGLE_UNLIKELY(result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR))
             {
                 // We get to ignore these as they are not fatal
@@ -579,11 +584,12 @@ VkResult CommandProcessor::getLastAndClearPresentResult(VkSwapchainKHR swapchain
     return result;
 }
 
-VkResult CommandProcessor::present(VkQueue queue, const VkPresentInfoKHR &presentInfo)
+VkResult CommandProcessor::present(egl::ContextPriority priority,
+                                   const VkPresentInfoKHR &presentInfo)
 {
     std::lock_guard<std::mutex> lock(mSwapchainStatusMutex);
     ANGLE_TRACE_EVENT0("gpu.angle", "vkQueuePresentKHR");
-    VkResult result = vkQueuePresentKHR(queue, &presentInfo);
+    VkResult result = mCommandQueue.queuePresent(priority, presentInfo);
 
     // Verify that we are presenting one and only one swapchain
     ASSERT(presentInfo.swapchainCount == 1);
@@ -602,6 +608,15 @@ CommandQueue::~CommandQueue() = default;
 
 void CommandQueue::destroy(RendererVk *renderer)
 {
+    // Force all commands to finish by flushing all queues.
+    for (VkQueue queue : mQueues)
+    {
+        if (queue != VK_NULL_HANDLE)
+        {
+            vkQueueWaitIdle(queue);
+        }
+    }
+
     mLastCompletedQueueSerial = Serial::Infinite();
     clearAllGarbage(renderer);
 
@@ -611,7 +626,7 @@ void CommandQueue::destroy(RendererVk *renderer)
     ASSERT(mInFlightCommands.empty() && mGarbageQueue.empty());
 }
 
-angle::Result CommandQueue::init(Context *context)
+angle::Result CommandQueue::init(Context *context, const DeviceQueueMap &queueMap)
 {
     RendererVk *renderer = context->getRenderer();
 
@@ -619,6 +634,8 @@ angle::Result CommandQueue::init(Context *context)
     uint32_t queueFamilyIndex = renderer->getQueueFamilyIndex();
     ANGLE_TRY(mPrimaryCommandPool.init(context, queueFamilyIndex));
 
+    mQueues = queueMap;
+
     return angle::Result::Continue;
 }
 
@@ -966,13 +983,18 @@ angle::Result CommandQueue::queueSubmit(Context *context,
         renderer->outputVmaStatString();
     }
 
-    VkQueue queue       = renderer->getVkQueue(contextPriority);
     VkFence fenceHandle = fence ? fence->getHandle() : VK_NULL_HANDLE;
-    ANGLE_VK_TRY(context, vkQueueSubmit(queue, 1, &submitInfo, fenceHandle));
+    ANGLE_VK_TRY(context, vkQueueSubmit(mQueues[contextPriority], 1, &submitInfo, fenceHandle));
     mLastSubmittedQueueSerial = submitQueueSerial;
 
     // Now that we've submitted work, clean up RendererVk garbage
     return renderer->cleanupGarbage(mLastCompletedQueueSerial);
 }
+
+VkResult CommandQueue::queuePresent(egl::ContextPriority contextPriority,
+                                    const VkPresentInfoKHR &presentInfo)
+{
+    return vkQueuePresentKHR(mQueues[contextPriority], &presentInfo);
+}
 }  // namespace vk
 }  // namespace rx

src/libANGLE/renderer/vulkan/CommandProcessor.h

@@ -65,7 +65,7 @@ class CommandProcessorTask
 
     void initProcessCommands(CommandBufferHelper *commandBuffer, const RenderPass *renderPass);
 
-    void initPresent(egl::ContextPriority priority, VkPresentInfoKHR &presentInfo);
+    void initPresent(egl::ContextPriority priority, const VkPresentInfoKHR &presentInfo);
 
     void initFinishToSerial(Serial serial);
 
@@ -157,13 +157,15 @@ struct CommandBatch final : angle::NonCopyable
     Serial serial;
 };
 
+using DeviceQueueMap = angle::PackedEnumMap<egl::ContextPriority, VkQueue>;
+
 class CommandQueue final : angle::NonCopyable
 {
   public:
     CommandQueue();
     ~CommandQueue();
 
-    angle::Result init(Context *context);
+    angle::Result init(Context *context, const DeviceQueueMap &queueMap);
     void destroy(RendererVk *renderer);
     void handleDeviceLost(RendererVk *renderer);
 
@@ -188,6 +190,8 @@ class CommandQueue final : angle::NonCopyable
                               const VkSubmitInfo &submitInfo,
                               const Fence *fence,
                               Serial submitQueueSerial);
+    VkResult queuePresent(egl::ContextPriority contextPriority,
+                          const VkPresentInfoKHR &presentInfo);
 
     angle::Result waitForSerialWithUserTimeout(vk::Context *context,
                                                Serial serial,
@@ -230,6 +234,9 @@ class CommandQueue final : angle::NonCopyable
     Serial mLastCompletedQueueSerial;
     Serial mLastSubmittedQueueSerial;
     Serial mCurrentQueueSerial;
+
+    // Devices queues.
+    DeviceQueueMap mQueues;
 };
 
 // TODO(jmadill): Give this the same API as CommandQueue. b/172704839
@@ -246,7 +253,7 @@ class CommandProcessor : public Context
 
     // Entry point for command processor thread, calls processTasksImpl to do the
     // work. called by Rendererinitialization on main thread
-    void processTasks();
+    void processTasks(const DeviceQueueMap &queueMap);
 
     // Called asynchronously from main thread to queue work that is then processed by the worker
     // thread
@@ -293,7 +300,7 @@ class CommandProcessor : public Context
     angle::Result processTask(CommandProcessorTask *task);
 
     VkResult getLastAndClearPresentResult(VkSwapchainKHR swapchain);
-    VkResult present(VkQueue queue, const VkPresentInfoKHR &presentInfo);
+    VkResult present(egl::ContextPriority priority, const VkPresentInfoKHR &presentInfo);
 
     std::queue<CommandProcessorTask> mTasks;
     mutable std::mutex mWorkerMutex;

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -504,16 +504,7 @@ void RendererVk::releaseSharedResources(vk::ResourceUseList *resourceList)
 
 void RendererVk::onDestroy(vk::Context *context)
 {
-    // Force all commands to finish by flushing all queues.
-    for (VkQueue queue : mQueues)
-    {
-        if (queue != VK_NULL_HANDLE)
-        {
-            vkQueueWaitIdle(queue);
-        }
-    }
-
-    if (getFeatures().asyncCommandQueue.enabled)
+    if (mFeatures.asyncCommandQueue.enabled)
     {
         // Shutdown worker thread
         mCommandProcessor.shutdown(&mCommandProcessorThread);
@@ -915,17 +906,6 @@ angle::Result RendererVk::initialize(DisplayVk *displayVk,
 
     setGlobalDebugAnnotator();
 
-    if (mFeatures.asyncCommandQueue.enabled)
-    {
-        mCommandProcessorThread =
-            std::thread(&vk::CommandProcessor::processTasks, &mCommandProcessor);
-        waitForCommandProcessorIdle(displayVk);
-    }
-    else
-    {
-        ANGLE_TRY(mCommandQueue.init(displayVk));
-    }
-
     return angle::Result::Continue;
 }
 
@@ -1480,11 +1460,13 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF
     mCurrentQueueFamilyIndex = queueFamilyIndex;
 
     // When only 1 Queue, use same for all, Low index. Identify as Medium, since it's default.
+    vk::DeviceQueueMap queueMap;
+
     VkQueue queue;
     vkGetDeviceQueue(mDevice, mCurrentQueueFamilyIndex, kQueueIndexLow, &queue);
-    mQueues[egl::ContextPriority::Low]        = queue;
-    mQueues[egl::ContextPriority::Medium]     = queue;
-    mQueues[egl::ContextPriority::High]       = queue;
+    queueMap[egl::ContextPriority::Low]       = queue;
+    queueMap[egl::ContextPriority::Medium]    = queue;
+    queueMap[egl::ContextPriority::High]      = queue;
     mPriorities[egl::ContextPriority::Low]    = egl::ContextPriority::Medium;
     mPriorities[egl::ContextPriority::Medium] = egl::ContextPriority::Medium;
     mPriorities[egl::ContextPriority::High]   = egl::ContextPriority::Medium;
@@ -1493,17 +1475,28 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF
     if (queueCount > 1)
     {
         vkGetDeviceQueue(mDevice, mCurrentQueueFamilyIndex, kQueueIndexHigh,
-                         &mQueues[egl::ContextPriority::High]);
+                         &queueMap[egl::ContextPriority::High]);
         mPriorities[egl::ContextPriority::High] = egl::ContextPriority::High;
     }
     // If at least 3 queues, Medium has its own queue. Adjust Low priority.
     if (queueCount > 2)
     {
         vkGetDeviceQueue(mDevice, mCurrentQueueFamilyIndex, kQueueIndexMedium,
-                         &mQueues[egl::ContextPriority::Medium]);
+                         &queueMap[egl::ContextPriority::Medium]);
         mPriorities[egl::ContextPriority::Low] = egl::ContextPriority::Low;
     }
 
+    if (mFeatures.asyncCommandQueue.enabled)
+    {
+        mCommandProcessorThread =
+            std::thread(&vk::CommandProcessor::processTasks, &mCommandProcessor, queueMap);
+        waitForCommandProcessorIdle(displayVk);
+    }
+    else
+    {
+        ANGLE_TRY(mCommandQueue.init(displayVk, queueMap));
+    }
+
 #if !defined(ANGLE_SHARED_LIBVULKAN)
     if (getFeatures().supportsTransformFeedbackExtension.enabled)
     {
@@ -2255,7 +2248,6 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
     else
     {
         std::lock_guard<std::mutex> commandQueueLock(mCommandQueueMutex);
-        std::lock_guard<std::mutex> queueLock(mQueueMutex);
 
         VkSubmitInfo submitInfo = {};
         submitInfo.sType        = VK_STRUCTURE_TYPE_SUBMIT_INFO;
@@ -2281,21 +2273,6 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
     return angle::Result::Continue;
 }
 
-VkResult RendererVk::queuePresent(egl::ContextPriority priority,
-                                  const VkPresentInfoKHR &presentInfo)
-{
-    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::queuePresent");
-
-    ASSERT(!mFeatures.asyncCommandQueue.enabled);
-
-    std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
-
-    {
-        ANGLE_TRACE_EVENT0("gpu.angle", "vkQueuePresentKHR");
-        return vkQueuePresentKHR(mQueues[priority], &presentInfo);
-    }
-}
-
 angle::Result RendererVk::newSharedFence(vk::Context *context,
                                          vk::Shared<vk::Fence> *sharedFenceOut)
 {
@@ -2558,7 +2535,6 @@ angle::Result RendererVk::submitFrame(vk::Context *context,
     else
     {
         std::lock_guard<std::mutex> commandQueueLock(mCommandQueueMutex);
-        std::lock_guard<std::mutex> queueLock(mQueueMutex);
 
         submitQueueSerial = mCommandQueue.reserveSubmitSerial();
 
@@ -2706,6 +2682,32 @@ angle::Result RendererVk::flushOutsideRPCommands(vk::Context *context,
     return angle::Result::Continue;
 }
 
+VkResult RendererVk::queuePresent(vk::Context *context,
+                                  egl::ContextPriority priority,
+                                  const VkPresentInfoKHR &presentInfo)
+{
+    VkResult result = VK_SUCCESS;
+    if (mFeatures.asyncCommandQueue.enabled)
+    {
+        vk::CommandProcessorTask present;
+        present.initPresent(priority, presentInfo);
+
+        ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present");
+        queueCommand(context, &present);
+
+        // Always return success, when we call acquireNextImage we'll check the return code. This
+        // allows the app to continue working until we really need to know the return code from
+        // present.
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        result = mCommandQueue.queuePresent(priority, presentInfo);
+    }
+
+    return result;
+}
+
 vk::CommandBufferHelper *RendererVk::getCommandBufferHelper(bool hasRenderPass)
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::getCommandBufferHelper");

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -171,9 +171,6 @@ class RendererVk : angle::NonCopyable
         return mPriorities[priority];
     }
 
-    // Queue submit that originates from the main thread
-    VkResult queuePresent(egl::ContextPriority priority, const VkPresentInfoKHR &presentInfo);
-
     // This command buffer should be submitted immediately via queueSubmitOneOff.
     angle::Result getCommandBufferOneOff(vk::Context *context,
                                          vk::PrimaryCommandBuffer *commandBufferOut);
@@ -298,7 +295,6 @@ class RendererVk : angle::NonCopyable
     }
 
     void finishAllWork(vk::Context *context) { mCommandProcessor.finishAllWork(context); }
-    VkQueue getVkQueue(egl::ContextPriority priority) const { return mQueues[priority]; }
 
     bool getEnableValidationLayers() const { return mEnableValidationLayers; }
 
@@ -335,6 +331,10 @@ class RendererVk : angle::NonCopyable
     angle::Result flushOutsideRPCommands(vk::Context *context,
                                          vk::CommandBufferHelper **outsideRPCommands);
 
+    VkResult queuePresent(vk::Context *context,
+                          egl::ContextPriority priority,
+                          const VkPresentInfoKHR &presentInfo);
+
     vk::CommandBufferHelper *getCommandBufferHelper(bool hasRenderPass);
     void recycleCommandBufferHelper(vk::CommandBufferHelper *commandBuffer);
 
@@ -396,8 +396,6 @@ class RendererVk : angle::NonCopyable
     VkExternalSemaphoreProperties mExternalSemaphoreProperties;
     VkPhysicalDeviceSamplerYcbcrConversionFeatures mSamplerYcbcrConversionFeatures;
     std::vector<VkQueueFamilyProperties> mQueueFamilyProperties;
-    std::mutex mQueueMutex;
-    angle::PackedEnumMap<egl::ContextPriority, VkQueue> mQueues;
     angle::PackedEnumMap<egl::ContextPriority, egl::ContextPriority> mPriorities;
     uint32_t mCurrentQueueFamilyIndex;
     uint32_t mMaxVertexAttribDivisor;

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -1403,23 +1403,7 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
     mCurrentSwapHistoryIndex =
         mCurrentSwapHistoryIndex == mSwapHistory.size() ? 0 : mCurrentSwapHistoryIndex;
 
-    VkResult result;
-    if (renderer->getFeatures().asyncCommandQueue.enabled)
-    {
-        vk::CommandProcessorTask present;
-        present.initPresent(contextVk->getPriority(), presentInfo);
-
-        ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present");
-        renderer->queueCommand(contextVk, &present);
-        // Always return success, when we call acquireNextImage we'll check the return code. This
-        // allows the app to continue working until we really need to know the return code from
-        // present.
-        result = VK_SUCCESS;
-    }
-    else
-    {
-        result = renderer->queuePresent(contextVk->getPriority(), presentInfo);
-    }
+    VkResult result = renderer->queuePresent(contextVk, contextVk->getPriority(), presentInfo);
 
     ANGLE_TRY(computePresentOutOfDate(contextVk, result, presentOutOfDate));