Vulkan: Use a persistent CommandPool

include/platform/FeaturesVk.h

@@ -184,6 +184,16 @@ struct FeaturesVk : FeatureSetBase
         "On platform with Intel or AMD gpu, vulkan swapchain is not returning VK_ERROR_OUT_OF_DATE"
         "when window resizing",
         &members, "http://anglebug.com/3623, http://anglebug.com/3624, http://anglebug.com/3625"};
+
+    // On Pixel1XL and Pixel2, reset a vkCommandBuffer seems to have side effects on binding
+    // descriptor sets to it afterwards, Work-around by keep using transient vkCommandBuffer on
+    // those devices.
+    // http://b/135763283
+    Feature transientCommandBuffer = {
+        "transient_command_buffer", FeatureCategory::VulkanWorkarounds,
+        "On Pixel2, keep using transient vkCommandBuffer to work around driver issue in reseting"
+        "vkCommandBuffer",
+        &members, "http://b/135763283"};
 };
 
 inline FeaturesVk::FeaturesVk()  = default;

src/libANGLE/renderer/vulkan/CommandGraph.cpp

@@ -44,9 +44,10 @@ angle::Result InitAndBeginCommandBuffer(vk::Context *context,
                                         const VkCommandBufferInheritanceInfo &inheritanceInfo,
                                         VkCommandBufferUsageFlags flags,
                                         angle::PoolAllocator *poolAllocator,
-                                        PrimaryCommandBuffer *commandBuffer)
+                                        priv::CommandBuffer *commandBuffer)
 {
     ASSERT(!commandBuffer->valid());
+    ASSERT(commandPool.valid());
     VkCommandBufferAllocateInfo createInfo = {};
     createInfo.sType                       = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
     createInfo.commandPool                 = commandPool.getHandle();
@@ -844,8 +845,7 @@ void CommandGraph::setNewBarrier(CommandGraphNode *newBarrier)
 angle::Result CommandGraph::submitCommands(ContextVk *context,
                                            Serial serial,
                                            RenderPassCache *renderPassCache,
-                                           CommandPool *commandPool,
-                                           PrimaryCommandBuffer *primaryCommandBufferOut)
+                                           PrimaryCommandBuffer *primaryCommandBuffer)
 {
     // There is no point in submitting an empty command buffer, so make sure not to call this
     // function if there's nothing to do.
@@ -862,14 +862,6 @@ angle::Result CommandGraph::submitCommands(ContextVk *context,
             previousBarrier, &mNodes[previousBarrierIndex + 1], afterNodesCount);
     }
 
-    VkCommandBufferAllocateInfo primaryInfo = {};
-    primaryInfo.sType                       = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
-    primaryInfo.commandPool                 = commandPool->getHandle();
-    primaryInfo.level                       = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
-    primaryInfo.commandBufferCount          = 1;
-
-    ANGLE_VK_TRY(context, primaryCommandBufferOut->init(context->getDevice(), primaryInfo));
-
     if (mEnableGraphDiagnostics)
     {
         dumpGraphDotFile(std::cout);
@@ -882,9 +874,9 @@ angle::Result CommandGraph::submitCommands(ContextVk *context,
     beginInfo.flags                    = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
     beginInfo.pInheritanceInfo         = nullptr;
 
-    ANGLE_VK_TRY(context, primaryCommandBufferOut->begin(beginInfo));
+    ANGLE_VK_TRY(context, primaryCommandBuffer->begin(beginInfo));
 
-    ANGLE_TRY(context->traceGpuEvent(primaryCommandBufferOut, TRACE_EVENT_PHASE_BEGIN,
+    ANGLE_TRY(context->traceGpuEvent(primaryCommandBuffer, TRACE_EVENT_PHASE_BEGIN,
                                      "Primary Command Buffer"));
 
     for (CommandGraphNode *topLevelNode : mNodes)
@@ -907,7 +899,7 @@ angle::Result CommandGraph::submitCommands(ContextVk *context,
                     break;
                 case VisitedState::Ready:
                     ANGLE_TRY(node->visitAndExecute(context, serial, renderPassCache,
-                                                    primaryCommandBufferOut));
+                                                    primaryCommandBuffer));
                     nodeStack.pop_back();
                     break;
                 case VisitedState::Visited:
@@ -920,10 +912,10 @@ angle::Result CommandGraph::submitCommands(ContextVk *context,
         }
     }
 
-    ANGLE_TRY(context->traceGpuEvent(primaryCommandBufferOut, TRACE_EVENT_PHASE_END,
+    ANGLE_TRY(context->traceGpuEvent(primaryCommandBuffer, TRACE_EVENT_PHASE_END,
                                      "Primary Command Buffer"));
 
-    ANGLE_VK_TRY(context, primaryCommandBufferOut->end());
+    ANGLE_VK_TRY(context, primaryCommandBuffer->end());
 
     clear();
 

src/libANGLE/renderer/vulkan/CommandGraph.h

@@ -482,8 +482,7 @@ class CommandGraph final : angle::NonCopyable
     angle::Result submitCommands(ContextVk *context,
                                  Serial serial,
                                  RenderPassCache *renderPassCache,
-                                 CommandPool *commandPool,
-                                 PrimaryCommandBuffer *primaryCommandBufferOut);
+                                 PrimaryCommandBuffer *primaryCommandBuffer);
     bool empty() const;
     void clear();
 

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -14,6 +14,7 @@
 
 #include "common/PackedEnums.h"
 #include "libANGLE/renderer/ContextImpl.h"
+#include "libANGLE/renderer/vulkan/PersistentCommandPool.h"
 #include "libANGLE/renderer/vulkan/RendererVk.h"
 #include "libANGLE/renderer/vulkan/vk_helpers.h"
 
@@ -456,7 +457,6 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
 
     angle::Result submitFrame(const VkSubmitInfo &submitInfo,
                               vk::PrimaryCommandBuffer &&commandBuffer);
-    void freeAllInFlightResources();
     angle::Result flushCommandGraph(vk::PrimaryCommandBuffer *commandBatch);
 
     angle::Result synchronizeCpuGpuTime();
@@ -561,6 +561,9 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
     vk::CommandPool mCommandPool;
     std::vector<vk::CommandPool> mCommandPoolFreeList;
 
+    // We use a Persistent CommandPool with pre-allocated buffers for primary CommandBuffer
+    vk::PersistentCommandPool mPrimaryCommandPool;
+
     Serial mLastCompletedQueueSerial;
     Serial mLastSubmittedQueueSerial;
     Serial mCurrentQueueSerial;
@@ -574,11 +577,17 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
 
         void destroy(VkDevice device);
 
+        vk::PrimaryCommandBuffer primaryCommands;
+        // commandPool is for secondary CommandBuffer allocation
         vk::CommandPool commandPool;
         vk::Shared<vk::Fence> fence;
         Serial serial;
     };
 
+    angle::Result releaseToCommandBatch(vk::PrimaryCommandBuffer &&commandBuffer,
+                                        CommandBatch *batch);
+    angle::Result recycleCommandBatch(CommandBatch *batch);
+
     std::vector<CommandBatch> mInFlightCommands;
     std::vector<vk::GarbageObject> mGarbage;
 

src/libANGLE/renderer/vulkan/PersistentCommandPool.cpp

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// PersistentCommandPool.cpp:
+//    Implements the class methods for PersistentCommandPool
+//
+
+#include "libANGLE/renderer/vulkan/PersistentCommandPool.h"
+
+namespace rx
+{
+
+namespace vk
+{
+
+PersistentCommandPool::PersistentCommandPool() {}
+
+PersistentCommandPool::~PersistentCommandPool()
+{
+    ASSERT(!mCommandPool.valid() && mFreeBuffers.empty());
+}
+
+angle::Result PersistentCommandPool::init(vk::Context *context, uint32_t queueFamilyIndex)
+{
+    ASSERT(!mCommandPool.valid());
+
+    // Initialize the command pool now that we know the queue family index.
+    VkCommandPoolCreateInfo commandPoolInfo = {};
+    commandPoolInfo.sType                   = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+    commandPoolInfo.flags                   = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
+    commandPoolInfo.queueFamilyIndex        = queueFamilyIndex;
+
+    ANGLE_VK_TRY(context, mCommandPool.init(context->getDevice(), commandPoolInfo));
+
+    for (uint32_t i = 0; i < kInitBufferNum; i++)
+    {
+        ANGLE_TRY(allocateCommandBuffer(context));
+    }
+
+    return angle::Result::Continue;
+}
+
+void PersistentCommandPool::destroy(const vk::Context *context)
+{
+    ASSERT(mCommandPool.valid());
+
+    VkDevice device = context->getDevice();
+    for (vk::PrimaryCommandBuffer &cmdBuf : mFreeBuffers)
+    {
+        cmdBuf.destroy(device, mCommandPool);
+    }
+    mFreeBuffers.clear();
+
+    mCommandPool.destroy(device);
+}
+
+angle::Result PersistentCommandPool::alloc(vk::Context *context,
+                                           vk::PrimaryCommandBuffer *bufferOutput)
+{
+    if (mFreeBuffers.empty())
+    {
+        ANGLE_TRY(allocateCommandBuffer(context));
+        ASSERT(!mFreeBuffers.empty());
+    }
+
+    *bufferOutput = std::move(mFreeBuffers.back());
+    mFreeBuffers.pop_back();
+
+    return angle::Result::Continue;
+}
+
+angle::Result PersistentCommandPool::collect(vk::Context *context,
+                                             vk::PrimaryCommandBuffer &&buffer)
+{
+    // VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT NOT set, The CommandBuffer
+    // can still hold the memory resource
+    ANGLE_VK_TRY(context, vkResetCommandBuffer(buffer.getHandle(), 0));
+
+    mFreeBuffers.emplace_back(std::move(buffer));
+    return angle::Result::Continue;
+}
+
+angle::Result PersistentCommandPool::allocateCommandBuffer(vk::Context *context)
+{
+    vk::PrimaryCommandBuffer commandBuffer;
+    {
+        // Only used for primary CommandBuffer allocation
+        VkCommandBufferAllocateInfo commandBufferInfo = {};
+        commandBufferInfo.sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+        commandBufferInfo.commandPool        = mCommandPool.getHandle();
+        commandBufferInfo.level              = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+        commandBufferInfo.commandBufferCount = 1;
+
+        ANGLE_VK_TRY(context, commandBuffer.init(context->getDevice(), commandBufferInfo));
+    }
+
+    mFreeBuffers.emplace_back(std::move(commandBuffer));
+
+    return angle::Result::Continue;
+}
+
+}  // namespace vk
+
+}  // namespace rx

src/libANGLE/renderer/vulkan/PersistentCommandPool.h

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// PersistentCommandPool.h:
+//    Defines the class interface for PersistentCommandBuffer
+//
+
+#ifndef LIBANGLE_RENDERER_VULKAN_PERSISTENTCOMMANDPOOL_H_
+#define LIBANGLE_RENDERER_VULKAN_PERSISTENTCOMMANDPOOL_H_
+
+#include <vector>
+
+#include "libANGLE/renderer/vulkan/vk_utils.h"
+#include "libANGLE/renderer/vulkan/vk_wrapper.h"
+
+namespace rx
+{
+
+namespace vk
+{
+
+class PersistentCommandPool final
+{
+  public:
+    PersistentCommandPool();
+    ~PersistentCommandPool();
+
+    void destroy(const vk::Context *context);
+    angle::Result init(vk::Context *context, uint32_t queueFamilyIndex);
+
+    angle::Result alloc(vk::Context *context, vk::PrimaryCommandBuffer *bufferOutput);
+    angle::Result collect(vk::Context *context, vk::PrimaryCommandBuffer &&buffer);
+
+    bool valid() const { return mCommandPool.valid(); }
+
+  private:
+    angle::Result allocateCommandBuffer(vk::Context *context);
+
+    std::vector<vk::PrimaryCommandBuffer> mFreeBuffers;
+
+    vk::CommandPool mCommandPool;
+
+    static const int kInitBufferNum = 2;
+};
+
+}  // namespace vk
+
+}  // namespace rx
+
+#endif

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -1273,6 +1273,12 @@ void RendererVk::initFeatures(const ExtensionNameList &deviceExtensionNames)
     {
         mFeatures.disableFlippingBlitWithCommand.enabled = true;
     }
+
+    if (IsPixel2(mPhysicalDeviceProperties.vendorID, mPhysicalDeviceProperties.deviceID) ||
+        IsPixel1XL(mPhysicalDeviceProperties.vendorID, mPhysicalDeviceProperties.deviceID))
+    {
+        mFeatures.transientCommandBuffer.enabled = true;
+    }
 }
 
 void RendererVk::initPipelineCacheVkKey()

src/libGLESv2.gni

@@ -862,6 +862,8 @@ libangle_vulkan_sources = [
   "src/libANGLE/renderer/vulkan/ImageVk.h",
   "src/libANGLE/renderer/vulkan/MemoryObjectVk.cpp",
   "src/libANGLE/renderer/vulkan/MemoryObjectVk.h",
+  "src/libANGLE/renderer/vulkan/PersistentCommandPool.cpp",
+  "src/libANGLE/renderer/vulkan/PersistentCommandPool.h",
   "src/libANGLE/renderer/vulkan/ProgramVk.cpp",
   "src/libANGLE/renderer/vulkan/ProgramVk.h",
   "src/libANGLE/renderer/vulkan/ProgramPipelineVk.cpp",