Vulkan: Move FenceRecyler to CommandQueue.

src/libANGLE/renderer/vulkan/CommandProcessor.cpp

@@ -59,6 +59,42 @@ bool CommandsHaveValidOrdering(const std::vector<vk::CommandBatch> &commands)
 }
 }  // namespace
 
+angle::Result FenceRecycler::newSharedFence(vk::Context *context,
+                                            vk::Shared<vk::Fence> *sharedFenceOut)
+{
+    bool gotRecycledFence = false;
+    vk::Fence fence;
+    {
+        std::lock_guard<std::mutex> lock(mMutex);
+        if (!mRecyler.empty())
+        {
+            mRecyler.fetch(&fence);
+            gotRecycledFence = true;
+        }
+    }
+
+    VkDevice device(context->getDevice());
+    if (gotRecycledFence)
+    {
+        ANGLE_VK_TRY(context, fence.reset(device));
+    }
+    else
+    {
+        VkFenceCreateInfo fenceCreateInfo = {};
+        fenceCreateInfo.sType             = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+        fenceCreateInfo.flags             = 0;
+        ANGLE_VK_TRY(context, fence.init(device, fenceCreateInfo));
+    }
+    sharedFenceOut->assign(device, std::move(fence));
+    return angle::Result::Continue;
+}
+
+void FenceRecycler::destroy(vk::Context *context)
+{
+    std::lock_guard<std::mutex> lock(mMutex);
+    mRecyler.destroy(context->getDevice());
+}
+
 // CommandProcessorTask implementation
 void CommandProcessorTask::initTask()
 {
@@ -381,16 +417,10 @@ angle::Result CommandProcessor::processTask(CommandProcessorTask *task)
             ANGLE_TRACE_EVENT0("gpu.angle", "processTask::FlushAndQueueSubmit");
             // End command buffer
 
-            // Get shared submit fence. It's possible there are other users of this fence that
-            // must wait for the work to be submitted before waiting on the fence. Reset the fence
-            // immediately so we are sure to get a fresh one next time.
-            Shared<Fence> fence;
-            ANGLE_TRY(mRenderer->newSharedFence(this, &fence));
-
             // Call submitFrame()
             ANGLE_TRY(mCommandQueue.submitFrame(
                 this, task->getPriority(), task->getWaitSemaphores(),
-                task->getWaitSemaphoreStageMasks(), task->getSemaphore(), std::move(fence),
+                task->getWaitSemaphoreStageMasks(), task->getSemaphore(),
                 std::move(task->getGarbage()), &mCommandPool, task->getQueueSerial()));
 
             ASSERT(task->getGarbage().empty());
@@ -603,7 +633,6 @@ angle::Result CommandProcessor::submitFrame(
     const std::vector<VkSemaphore> &waitSemaphores,
     const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
     const Semaphore *signalSemaphore,
-    Shared<Fence> &&sharedFence,
     GarbageList &&currentGarbage,
     CommandPool *commandPool,
     Serial submitQueueSerial)
@@ -711,6 +740,7 @@ void CommandQueue::destroy(Context *context)
 
     mPrimaryCommands.destroy(renderer->getDevice());
     mPrimaryCommandPool.destroy(renderer->getDevice());
+    mFenceRecycler.destroy(context);
 
     ASSERT(mInFlightCommands.empty() && mGarbageQueue.empty());
 }
@@ -767,7 +797,7 @@ angle::Result CommandQueue::retireFinishedCommands(Context *context, size_t fini
         CommandBatch &batch = mInFlightCommands[commandIndex];
 
         mLastCompletedQueueSerial = batch.serial;
-        renderer->resetSharedFence(&batch.fence);
+        mFenceRecycler.resetSharedFence(&batch.fence);
         ANGLE_TRACE_EVENT0("gpu.angle", "command buffer recycling");
         batch.commandPool.destroy(device);
         ANGLE_TRY(mPrimaryCommandPool.collect(context, std::move(batch.primaryCommands)));
@@ -924,7 +954,6 @@ angle::Result CommandQueue::submitFrame(
     const std::vector<VkSemaphore> &waitSemaphores,
     const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
     const Semaphore *signalSemaphore,
-    Shared<Fence> &&sharedFence,
     GarbageList &&currentGarbage,
     CommandPool *commandPool,
     Serial submitQueueSerial)
@@ -944,8 +973,9 @@ angle::Result CommandQueue::submitFrame(
 
     DeviceScoped<CommandBatch> scopedBatch(device);
     CommandBatch &batch = scopedBatch.get();
-    batch.fence         = std::move(sharedFence);
-    batch.serial        = submitQueueSerial;
+
+    ANGLE_TRY(mFenceRecycler.newSharedFence(context, &batch.fence));
+    batch.serial = submitQueueSerial;
 
     ANGLE_TRY(queueSubmit(context, priority, submitInfo, &batch.fence.get(), batch.serial));
 

src/libANGLE/renderer/vulkan/CommandProcessor.h

@@ -27,6 +27,25 @@ class CommandProcessor;
 
 namespace vk
 {
+class FenceRecycler
+{
+  public:
+    FenceRecycler() {}
+    ~FenceRecycler() {}
+    void destroy(vk::Context *context);
+
+    angle::Result newSharedFence(vk::Context *context, vk::Shared<vk::Fence> *sharedFenceOut);
+    inline void resetSharedFence(vk::Shared<vk::Fence> *sharedFenceIn)
+    {
+        std::lock_guard<std::mutex> lock(mMutex);
+        sharedFenceIn->resetAndRecycle(&mRecyler);
+    }
+
+  private:
+    std::mutex mMutex;
+    vk::Recycler<vk::Fence> mRecyler;
+};
+
 enum class CustomTask
 {
     Invalid = 0,
@@ -175,7 +194,6 @@ class CommandQueueInterface : angle::NonCopyable
         const std::vector<VkSemaphore> &waitSemaphores,
         const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
         const Semaphore *signalSemaphore,
-        Shared<Fence> &&sharedFence,
         GarbageList &&currentGarbage,
         CommandPool *commandPool,
         Serial submitQueueSerial)                                      = 0;
@@ -229,7 +247,6 @@ class CommandQueue final : public CommandQueueInterface
                               const std::vector<VkSemaphore> &waitSemaphores,
                               const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
                               const Semaphore *signalSemaphore,
-                              Shared<Fence> &&sharedFence,
                               GarbageList &&currentGarbage,
                               CommandPool *commandPool,
                               Serial submitQueueSerial) override;
@@ -291,6 +308,8 @@ class CommandQueue final : public CommandQueueInterface
 
     // Devices queues.
     DeviceQueueMap mQueues;
+
+    FenceRecycler mFenceRecycler;
 };
 
 // CommandProcessor is used to dispatch work to the GPU when the asyncCommandQueue feature is
@@ -336,7 +355,6 @@ class CommandProcessor : public Context, public CommandQueueInterface
                               const std::vector<VkSemaphore> &waitSemaphores,
                               const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
                               const Semaphore *signalSemaphore,
-                              Shared<Fence> &&sharedFence,
                               GarbageList &&currentGarbage,
                               CommandPool *commandPool,
                               Serial submitQueueSerial) override;

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -534,11 +534,6 @@ void RendererVk::onDestroy(vk::Context *context)
 
     mOneOffCommandPool.destroy(mDevice);
 
-    {
-        std::lock_guard<std::mutex> lock(mFenceRecyclerMutex);
-        mFenceRecycler.destroy(mDevice);
-    }
-
     mPipelineCache.destroy(mDevice);
     mSamplerCache.destroy(this);
     mYuvConversionCache.destroy(this);
@@ -2251,34 +2246,6 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
     return angle::Result::Continue;
 }
 
-angle::Result RendererVk::newSharedFence(vk::Context *context,
-                                         vk::Shared<vk::Fence> *sharedFenceOut)
-{
-    bool gotRecycledFence = false;
-    vk::Fence fence;
-    {
-        std::lock_guard<std::mutex> lock(mFenceRecyclerMutex);
-        if (!mFenceRecycler.empty())
-        {
-            mFenceRecycler.fetch(&fence);
-            gotRecycledFence = true;
-        }
-    }
-    if (gotRecycledFence)
-    {
-        ANGLE_VK_TRY(context, fence.reset(mDevice));
-    }
-    else
-    {
-        VkFenceCreateInfo fenceCreateInfo = {};
-        fenceCreateInfo.sType             = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-        fenceCreateInfo.flags             = 0;
-        ANGLE_VK_TRY(context, fence.init(mDevice, fenceCreateInfo));
-    }
-    sharedFenceOut->assign(mDevice, std::move(fence));
-    return angle::Result::Continue;
-}
-
 template <VkFormatFeatureFlags VkFormatProperties::*features>
 VkFormatFeatureFlags RendererVk::getFormatFeatureBits(VkFormat format,
                                                       const VkFormatFeatureFlags featureBits) const
@@ -2455,9 +2422,6 @@ angle::Result RendererVk::submitFrame(vk::Context *context,
 {
     std::lock_guard<std::mutex> commandQueueLock(mCommandQueueMutex);
 
-    vk::Shared<vk::Fence> submitFence;
-    ANGLE_TRY(newSharedFence(context, &submitFence));
-
     Serial submitQueueSerial;
 
     if (mFeatures.asyncCommandQueue.enabled)
@@ -2466,7 +2430,7 @@ angle::Result RendererVk::submitFrame(vk::Context *context,
 
         ANGLE_TRY(mCommandProcessor.submitFrame(
             context, contextPriority, waitSemaphores, waitSemaphoreStageMasks, signalSemaphore,
-            std::move(submitFence), std::move(currentGarbage), commandPool, submitQueueSerial));
+            std::move(currentGarbage), commandPool, submitQueueSerial));
     }
     else
     {
@@ -2474,7 +2438,7 @@ angle::Result RendererVk::submitFrame(vk::Context *context,
 
         ANGLE_TRY(mCommandQueue.submitFrame(
             context, contextPriority, waitSemaphores, waitSemaphoreStageMasks, signalSemaphore,
-            std::move(submitFence), std::move(currentGarbage), commandPool, submitQueueSerial));
+            std::move(currentGarbage), commandPool, submitQueueSerial));
     }
 
     waitSemaphores.clear();

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -183,13 +183,6 @@ class RendererVk : angle::NonCopyable
                                     const vk::Fence *fence,
                                     Serial *serialOut);
 
-    angle::Result newSharedFence(vk::Context *context, vk::Shared<vk::Fence> *sharedFenceOut);
-    inline void resetSharedFence(vk::Shared<vk::Fence> *sharedFenceIn)
-    {
-        std::lock_guard<std::mutex> lock(mFenceRecyclerMutex);
-        sharedFenceIn->resetAndRecycle(&mFenceRecycler);
-    }
-
     template <typename... ArgsT>
     void collectGarbageAndReinit(vk::SharedResourceUse *use, ArgsT... garbageIn)
     {
@@ -396,9 +389,6 @@ class RendererVk : angle::NonCopyable
 
     bool mDeviceLost;
 
-    std::mutex mFenceRecyclerMutex;
-    vk::Recycler<vk::Fence> mFenceRecycler;
-
     std::mutex mGarbageMutex;
     vk::SharedGarbageList mSharedGarbage;