Vulkan: Recycle vkFence

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -667,7 +667,7 @@ angle::Result ContextVk::submitFrame(const VkSubmitInfo &submitInfo,
     mInFlightCommands.emplace_back(scopedBatch.release());
 
     // Make sure a new fence is created for the next submission.
-    mSubmitFence.reset(device);
+    mRenderer->resetSharedFence(&mSubmitFence);
 
     // CPU should be throttled to avoid mInFlightCommands from growing too fast.  That is done on
     // swap() though, and there could be multiple submissions in between (through glFlush() calls),
@@ -2272,17 +2272,12 @@ angle::Result ContextVk::getRenderPassWithOps(const vk::RenderPassDesc &desc,
 
 angle::Result ContextVk::getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOut)
 {
-    VkDevice device = getDevice();
     if (!mSubmitFence.isReferenced())
     {
-        vk::Fence fence;
-        VkFenceCreateInfo fenceCreateInfo = {};
-        fenceCreateInfo.sType             = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-        fenceCreateInfo.flags             = 0;
-        ANGLE_VK_TRY(this, fence.init(device, fenceCreateInfo));
-        mSubmitFence.assign(device, std::move(fence));
+        ANGLE_TRY(getRenderer()->newSharedFence(this, &mSubmitFence));
     }
-    sharedFenceOut->copy(device, mSubmitFence);
+    getRenderer()->resetSharedFence(sharedFenceOut);
+    sharedFenceOut->copy(getDevice(), mSubmitFence);
     return angle::Result::Continue;
 }
 

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -473,7 +473,7 @@ angle::Result WaitFences(vk::Context *context,
         }
         ANGLE_VK_TRY(context, result);
 
-        fences->back().reset(context->getDevice());
+        context->getRenderer()->resetSharedFence(&fences->back());
         fences->pop_back();
     }
 
@@ -513,6 +513,8 @@ void RendererVk::onDestroy(vk::Context *context)
 {
     (void)cleanupGarbage(context, true);
 
+    mFenceRecycler.destroy(mDevice);
+
     mPipelineLayoutCache.destroy(mDevice);
     mDescriptorSetLayoutCache.destroy(mDevice);
 
@@ -1477,6 +1479,26 @@ Serial RendererVk::nextSerial()
     return mQueueSerialFactory.generate();
 }
 
+angle::Result RendererVk::newSharedFence(vk::Context *context,
+                                         vk::Shared<vk::Fence> *sharedFenceOut)
+{
+    vk::Fence fence;
+    if (mFenceRecycler.empty())
+    {
+        VkFenceCreateInfo fenceCreateInfo = {};
+        fenceCreateInfo.sType             = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+        fenceCreateInfo.flags             = 0;
+        ANGLE_VK_TRY(context, fence.init(mDevice, fenceCreateInfo));
+    }
+    else
+    {
+        mFenceRecycler.fetch(mDevice, &fence);
+        ANGLE_VK_TRY(context, fence.reset(mDevice));
+    }
+    sharedFenceOut->assign(mDevice, std::move(fence));
+    return angle::Result::Continue;
+}
+
 void RendererVk::addGarbage(vk::Shared<vk::Fence> &&fence,
                             std::vector<vk::GarbageObjectBase> &&garbage)
 {

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -145,6 +145,12 @@ class RendererVk : angle::NonCopyable
 
     Serial nextSerial();
 
+    angle::Result newSharedFence(vk::Context *context, vk::Shared<vk::Fence> *sharedFenceOut);
+    inline void resetSharedFence(vk::Shared<vk::Fence> *sharedFenceIn)
+    {
+        sharedFenceIn->resetAndRecycle(&mFenceRecycler);
+    }
+
     void addGarbage(vk::Shared<vk::Fence> &&fence, std::vector<vk::GarbageObjectBase> &&garbage);
     void addGarbage(std::vector<vk::Shared<vk::Fence>> &&fences,
                     std::vector<vk::GarbageObjectBase> &&garbage);
@@ -199,6 +205,8 @@ class RendererVk : angle::NonCopyable
 
     bool mDeviceLost;
 
+    vk::Recycler<vk::Fence> mFenceRecycler;
+
     std::mutex mGarbageMutex;
     using FencedGarbage =
         std::pair<std::vector<vk::Shared<vk::Fence>>, std::vector<vk::GarbageObjectBase>>;

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -334,16 +334,16 @@ WindowSurfaceVk::SwapHistory::SwapHistory() = default;
 
 WindowSurfaceVk::SwapHistory::~SwapHistory() = default;
 
-void WindowSurfaceVk::SwapHistory::destroy(VkDevice device)
+void WindowSurfaceVk::SwapHistory::destroy(RendererVk *renderer)
 {
     if (swapchain != VK_NULL_HANDLE)
     {
-        vkDestroySwapchainKHR(device, swapchain, nullptr);
+        vkDestroySwapchainKHR(renderer->getDevice(), swapchain, nullptr);
         swapchain = VK_NULL_HANDLE;
     }
 
-    sharedFence.reset(device);
-    presentImageSemaphore.destroy(device);
+    renderer->resetSharedFence(&sharedFence);
+    presentImageSemaphore.destroy(renderer->getDevice());
 }
 
 angle::Result WindowSurfaceVk::SwapHistory::waitFence(ContextVk *contextVk)
@@ -399,7 +399,7 @@ void WindowSurfaceVk::destroy(const egl::Display *display)
 
     for (SwapHistory &swap : mSwapHistory)
     {
-        swap.destroy(device);
+        swap.destroy(renderer);
     }
 
     if (mSwapchain)
@@ -859,7 +859,7 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
     {
         ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present: Throttle CPU");
         ANGLE_TRY(swap.waitFence(contextVk));
-        swap.destroy(contextVk->getDevice());
+        swap.destroy(contextVk->getRenderer());
     }
 
     SwapchainImage &image = mSwapchainImages[mCurrentSwapchainImageIndex];

src/libANGLE/renderer/vulkan/SurfaceVk.h

@@ -212,7 +212,7 @@ class WindowSurfaceVk : public SurfaceVk
         SwapHistory &operator=(SwapHistory &&other) = delete;
         ~SwapHistory();
 
-        void destroy(VkDevice device);
+        void destroy(RendererVk *renderer);
 
         angle::Result waitFence(ContextVk *contextVk);
 

src/libANGLE/renderer/vulkan/vk_utils.h

@@ -459,6 +459,23 @@ class Shared final : angle::NonCopyable
 
     void reset(VkDevice device) { set(device, nullptr); }
 
+    template <typename RecyclerT>
+    void resetAndRecycle(RecyclerT *recycler)
+    {
+        if (mRefCounted)
+        {
+            mRefCounted->releaseRef();
+            if (!mRefCounted->isReferenced())
+            {
+                ASSERT(mRefCounted->get().valid());
+                recycler->recyle(std::move(mRefCounted->get()));
+                SafeDelete(mRefCounted);
+            }
+
+            mRefCounted = nullptr;
+        }
+    }
+
     bool isReferenced() const
     {
         // If reference is zero, the object should have been deleted.  I.e. if the object is not
@@ -482,6 +499,35 @@ class Shared final : angle::NonCopyable
     RefCounted<T> *mRefCounted;
 };
 
+template <typename T>
+class Recycler final : angle::NonCopyable
+{
+  public:
+    Recycler() = default;
+
+    void recyle(T &&garbageObject) { mObjectFreeList.emplace_back(std::move(garbageObject)); }
+
+    void fetch(VkDevice device, T *outObject)
+    {
+        ASSERT(!empty());
+        *outObject = std::move(mObjectFreeList.back());
+        mObjectFreeList.pop_back();
+    }
+
+    void destroy(VkDevice device)
+    {
+        for (T &object : mObjectFreeList)
+        {
+            object.destroy(device);
+        }
+    }
+
+    bool empty() const { return mObjectFreeList.empty(); }
+
+  private:
+    std::vector<T> mObjectFreeList;
+};
+
 }  // namespace vk
 
 // List of function pointers for used extensions.

src/libANGLE/renderer/vulkan/vk_wrapper.h

@@ -534,6 +534,7 @@ class Fence final : public WrappedObject<Fence, VkFence>
     using WrappedObject::operator=;
 
     VkResult init(VkDevice device, const VkFenceCreateInfo &createInfo);
+    VkResult reset(VkDevice device);
     VkResult getStatus(VkDevice device) const;
     VkResult wait(VkDevice device, uint64_t timeout) const;
 };
@@ -1398,6 +1399,12 @@ ANGLE_INLINE VkResult Fence::init(VkDevice device, const VkFenceCreateInfo &crea
     return vkCreateFence(device, &createInfo, nullptr, &mHandle);
 }
 
+ANGLE_INLINE VkResult Fence::reset(VkDevice device)
+{
+    ASSERT(valid());
+    return vkResetFences(device, 1, &mHandle);
+}
+
 ANGLE_INLINE VkResult Fence::getStatus(VkDevice device) const
 {
     ASSERT(valid());

src/tests/egl_tests/EGLSurfaceTest.cpp

@@ -356,6 +356,7 @@ TEST_P(EGLSurfaceTest, SwapInterval)
     // Flaky hang on Nexus 5X and 6P. http://anglebug.com/3364
     ANGLE_SKIP_TEST_IF((IsNexus5X() || IsNexus6P()) && isGLESRenderer());
     // Flaky hang on Ubuntu 19.04 NVIDIA Vulkan. http://anglebug.com/3618
+    // Maybe hang due to bug in NVIDIA Linux Vulkan driver. http://anglebug.com/3450
     ANGLE_SKIP_TEST_IF(IsLinux() && IsNVIDIA() && isVulkanRenderer());
 
     initializeDisplay();