Vulkan: Move CommandBuffer management to RendererVk.

src/libANGLE/renderer/vulkan/CommandProcessor.cpp

@@ -62,7 +62,6 @@ bool CommandsHaveValidOrdering(const std::vector<vk::CommandBatch> &commands)
 void CommandProcessorTask::initTask()
 {
     mTask                        = CustomTask::Invalid;
-    mContextVk                   = nullptr;
     mRenderPass                  = nullptr;
     mCommandBuffer               = nullptr;
     mSemaphore                   = nullptr;
@@ -77,12 +76,10 @@ void CommandProcessorTask::initTask()
 }
 
 // CommandProcessorTask implementation
-void CommandProcessorTask::initProcessCommands(ContextVk *contextVk,
-                                               CommandBufferHelper *commandBuffer,
+void CommandProcessorTask::initProcessCommands(CommandBufferHelper *commandBuffer,
                                                const RenderPass *renderPass)
 {
     mTask          = CustomTask::ProcessCommands;
-    mContextVk     = contextVk;
     mCommandBuffer = commandBuffer;
     mRenderPass    = renderPass;
 }
@@ -202,7 +199,6 @@ CommandProcessorTask &CommandProcessorTask::operator=(CommandProcessorTask &&rhs
         return *this;
     }
 
-    mContextVk     = rhs.mContextVk;
     mRenderPass    = rhs.mRenderPass;
     mCommandBuffer = rhs.mCommandBuffer;
     std::swap(mTask, rhs.mTask);
@@ -467,7 +463,7 @@ angle::Result CommandProcessor::processTask(CommandProcessorTask *task)
                 ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(this, task->getCommandBuffer()));
             }
             ASSERT(task->getCommandBuffer()->empty());
-            task->getCommandBuffer()->releaseToContextQueue(task->getContextVk());
+            mRenderer->recycleCommandBufferHelper(task->getCommandBuffer());
             break;
         }
         case CustomTask::CheckCompletedCommands:

src/libANGLE/renderer/vulkan/CommandProcessor.h

@@ -63,9 +63,7 @@ class CommandProcessorTask
 
     void initTask(CustomTask command) { mTask = command; }
 
-    void initProcessCommands(ContextVk *contextVk,
-                             CommandBufferHelper *commandBuffer,
-                             const RenderPass *renderPass);
+    void initProcessCommands(CommandBufferHelper *commandBuffer, const RenderPass *renderPass);
 
     void initPresent(egl::ContextPriority priority, VkPresentInfoKHR &presentInfo);
 
@@ -106,7 +104,6 @@ class CommandProcessorTask
     const VkPresentInfoKHR &getPresentInfo() const { return mPresentInfo; }
     const RenderPass *getRenderPass() const { return mRenderPass; }
     CommandBufferHelper *getCommandBuffer() const { return mCommandBuffer; }
-    ContextVk *getContextVk() const { return mContextVk; }
 
   private:
     void copyPresentInfo(const VkPresentInfoKHR &other);
@@ -114,7 +111,6 @@ class CommandProcessorTask
     CustomTask mTask;
 
     // ProcessCommands
-    ContextVk *mContextVk;
     const RenderPass *mRenderPass;
     CommandBufferHelper *mCommandBuffer;
 

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -505,6 +505,12 @@ void ContextVk::onDestroy(const gl::Context *context)
         queryPool.destroy(device);
     }
 
+    // Recycle current commands buffers.
+    mRenderer->recycleCommandBufferHelper(mOutsideRenderPassCommands);
+    mRenderer->recycleCommandBufferHelper(mRenderPassCommands);
+    mOutsideRenderPassCommands = nullptr;
+    mRenderPassCommands        = nullptr;
+
     ASSERT(mCurrentGarbage.empty());
 
     mRenderer->releaseSharedResources(&mResourceUseList);
@@ -612,17 +618,9 @@ angle::Result ContextVk::initialize()
 
     mUseOldRewriteStructSamplers = shouldUseOldRewriteStructSamplers();
 
-    // Prepare command buffer queue by:
-    //  1. Initializing each command buffer (as non-renderpass initially)
-    //  2. Put a pointer to each command buffer into queue
-    for (vk::CommandBufferHelper &commandBuffer : mCommandBuffers)
-    {
-        commandBuffer.initialize(false);
-        recycleCommandBuffer(&commandBuffer);
-    }
-    // Now assign initial command buffers from queue
-    getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
-    getNextAvailableCommandBuffer(&mRenderPassCommands, true);
+    // Assign initial command buffers from queue
+    mOutsideRenderPassCommands = mRenderer->getCommandBufferHelper(false);
+    mRenderPassCommands        = mRenderer->getCommandBufferHelper(true);
 
     if (mGpuEventsEnabled)
     {
@@ -4548,12 +4546,6 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
 
     ANGLE_TRY(mRenderer->flushRenderPassCommands(this, *renderPass, &mRenderPassCommands));
 
-    // TODO(jmadill): Manage in RendererVk. b/172678125
-    if (getFeatures().commandProcessor.enabled)
-    {
-        getNextAvailableCommandBuffer(&mRenderPassCommands, true);
-    }
-
     if (mGpuEventsEnabled)
     {
         EventName eventName = GetTraceEventName("RP", mPerfCounters.renderPasses);
@@ -4571,31 +4563,6 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
     return angle::Result::Continue;
 }
 
-void ContextVk::getNextAvailableCommandBuffer(vk::CommandBufferHelper **commandBuffer,
-                                              bool hasRenderPass)
-{
-    ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::getNextAvailableCommandBuffer");
-    std::unique_lock<std::mutex> lock(mCommandBufferQueueMutex);
-    // Only wake if notified and command queue is not empty
-    mAvailableCommandBufferCondition.wait(lock,
-                                          [this] { return !mAvailableCommandBuffers.empty(); });
-    *commandBuffer = mAvailableCommandBuffers.front();
-    ASSERT((*commandBuffer)->empty());
-    mAvailableCommandBuffers.pop();
-    lock.unlock();
-    (*commandBuffer)->setHasRenderPass(hasRenderPass);
-    (*commandBuffer)->markOpen();
-}
-
-void ContextVk::recycleCommandBuffer(vk::CommandBufferHelper *commandBuffer)
-{
-    ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::recycleCommandBuffer");
-    std::lock_guard<std::mutex> queueLock(mCommandBufferQueueMutex);
-    ASSERT(commandBuffer->empty());
-    mAvailableCommandBuffers.push(commandBuffer);
-    mAvailableCommandBufferCondition.notify_one();
-}
-
 angle::Result ContextVk::syncExternalMemory()
 {
     VkMemoryBarrier memoryBarrier = {};
@@ -4687,12 +4654,6 @@ angle::Result ContextVk::flushOutsideRenderPassCommands()
 
     ANGLE_TRY(mRenderer->flushOutsideRPCommands(this, &mOutsideRenderPassCommands));
 
-    // TODO(jmadill): Manage in RendererVk. b/172678125
-    if (getFeatures().commandProcessor.enabled)
-    {
-        getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
-    }
-
     mPerfCounters.flushedOutsideRenderPassCommandBuffers++;
     return angle::Result::Continue;
 }

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -592,9 +592,6 @@ class ContextVk : public ContextImpl, public vk::Context
     void updateOverlayOnPresent();
     void addOverlayUsedBuffersCount(vk::CommandBufferHelper *commandBuffer);
 
-    // When worker thread completes, it releases command buffers back to context queue
-    void recycleCommandBuffer(vk::CommandBufferHelper *commandBuffer);
-
     // DescriptorSet writes
     VkDescriptorBufferInfo *allocDescriptorBufferInfos(size_t count);
     VkDescriptorImageInfo *allocDescriptorImageInfos(size_t count);
@@ -904,9 +901,6 @@ class ContextVk : public ContextImpl, public vk::Context
 
     void initIndexTypeMap();
 
-    // Pull an available CBH ptr from the CBH queue and set to specified hasRenderPass state
-    void getNextAvailableCommandBuffer(vk::CommandBufferHelper **commandBuffer, bool hasRenderPass);
-
     angle::Result endRenderPassIfImageUsed(const vk::ImageHelper &image);
 
     angle::Result endRenderPassIfTransformFeedbackBuffer(const vk::BufferHelper *buffer);
@@ -1031,17 +1025,6 @@ class ContextVk : public ContextImpl, public vk::Context
 
     RenderPassCache mRenderPassCache;
 
-    // We have a queue of CommandBufferHelpers (CBHs) that is drawn from for the two active command
-    //  buffers in the main thread. The two active command buffers are the inside and outside
-    //  RenderPass command buffers.
-    constexpr static size_t kNumCommandBuffers = 50;
-    std::array<vk::CommandBufferHelper, kNumCommandBuffers> mCommandBuffers;
-
-    // Lock access to the command buffer queue
-    std::mutex mCommandBufferQueueMutex;
-    std::queue<vk::CommandBufferHelper *> mAvailableCommandBuffers;
-    std::condition_variable mAvailableCommandBufferCondition;
-
     vk::CommandBufferHelper *mOutsideRenderPassCommands;
     vk::CommandBufferHelper *mRenderPassCommands;
 

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -548,6 +548,12 @@ void RendererVk::onDestroy(vk::Context *context)
     mSamplerCache.destroy(this);
     mYuvConversionCache.destroy(this);
 
+    for (vk::CommandBufferHelper *commandBufferHelper : mCommandBufferHelperFreeList)
+    {
+        SafeDelete(commandBufferHelper);
+    }
+    mCommandBufferHelperFreeList.clear();
+
     mAllocator.destroy();
 
     if (mGlslangInitialized)
@@ -2666,7 +2672,7 @@ angle::Result RendererVk::checkCompletedCommands(vk::Context *context)
     return angle::Result::Continue;
 }
 
-angle::Result RendererVk::flushRenderPassCommands(ContextVk *contextVk,
+angle::Result RendererVk::flushRenderPassCommands(vk::Context *context,
                                                   const vk::RenderPass &renderPass,
                                                   vk::CommandBufferHelper **renderPassCommands)
 {
@@ -2675,20 +2681,20 @@ angle::Result RendererVk::flushRenderPassCommands(ContextVk *contextVk,
     {
         (*renderPassCommands)->markClosed();
         vk::CommandProcessorTask flushToPrimary;
-        flushToPrimary.initProcessCommands(contextVk, *renderPassCommands, &renderPass);
-        commandProcessorSyncErrorsAndQueueCommand(contextVk, &flushToPrimary);
+        flushToPrimary.initProcessCommands(*renderPassCommands, &renderPass);
+        commandProcessorSyncErrorsAndQueueCommand(context, &flushToPrimary);
+        *renderPassCommands = getCommandBufferHelper(true);
     }
     else
     {
         std::lock_guard<std::mutex> lock(mCommandQueueMutex);
-        ANGLE_TRY(
-            mCommandQueue.flushRenderPassCommands(contextVk, renderPass, *renderPassCommands));
+        ANGLE_TRY(mCommandQueue.flushRenderPassCommands(context, renderPass, *renderPassCommands));
     }
 
     return angle::Result::Continue;
 }
 
-angle::Result RendererVk::flushOutsideRPCommands(ContextVk *contextVk,
+angle::Result RendererVk::flushOutsideRPCommands(vk::Context *context,
                                                  vk::CommandBufferHelper **outsideRPCommands)
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::flushOutsideRPCommands");
@@ -2696,16 +2702,46 @@ angle::Result RendererVk::flushOutsideRPCommands(ContextVk *contextVk,
     {
         (*outsideRPCommands)->markClosed();
         vk::CommandProcessorTask flushToPrimary;
-        flushToPrimary.initProcessCommands(contextVk, *outsideRPCommands, nullptr);
-        commandProcessorSyncErrorsAndQueueCommand(contextVk, &flushToPrimary);
+        flushToPrimary.initProcessCommands(*outsideRPCommands, nullptr);
+        commandProcessorSyncErrorsAndQueueCommand(context, &flushToPrimary);
+        *outsideRPCommands = getCommandBufferHelper(false);
     }
     else
     {
         std::lock_guard<std::mutex> lock(mCommandQueueMutex);
-        ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(contextVk, *outsideRPCommands));
+        ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(context, *outsideRPCommands));
     }
 
     return angle::Result::Continue;
 }
 
+vk::CommandBufferHelper *RendererVk::getCommandBufferHelper(bool hasRenderPass)
+{
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::getCommandBufferHelper");
+    std::unique_lock<std::mutex> lock(mCommandBufferHelperFreeListMutex);
+
+    if (mCommandBufferHelperFreeList.empty())
+    {
+        vk::CommandBufferHelper *commandBuffer = new vk::CommandBufferHelper();
+        commandBuffer->initialize(hasRenderPass);
+        return commandBuffer;
+    }
+    else
+    {
+        vk::CommandBufferHelper *commandBuffer = mCommandBufferHelperFreeList.back();
+        mCommandBufferHelperFreeList.pop_back();
+        commandBuffer->setHasRenderPass(hasRenderPass);
+        return commandBuffer;
+    }
+}
+
+void RendererVk::recycleCommandBufferHelper(vk::CommandBufferHelper *commandBuffer)
+{
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::recycleCommandBufferHelper");
+    std::lock_guard<std::mutex> lock(mCommandBufferHelperFreeListMutex);
+
+    ASSERT(commandBuffer->empty());
+    commandBuffer->markOpen();
+    mCommandBufferHelperFreeList.push_back(commandBuffer);
+}
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -329,13 +329,15 @@ class RendererVk : angle::NonCopyable
     angle::Result finish(vk::Context *context);
     angle::Result checkCompletedCommands(vk::Context *context);
 
-    // TODO(jmadill): Use vk::Context instead of ContextVk. b/172704839
-    angle::Result flushRenderPassCommands(ContextVk *contextVk,
+    angle::Result flushRenderPassCommands(vk::Context *context,
                                           const vk::RenderPass &renderPass,
                                           vk::CommandBufferHelper **renderPassCommands);
-    angle::Result flushOutsideRPCommands(ContextVk *contextVk,
+    angle::Result flushOutsideRPCommands(vk::Context *context,
                                          vk::CommandBufferHelper **outsideRPCommands);
 
+    vk::CommandBufferHelper *getCommandBufferHelper(bool hasRenderPass);
+    void recycleCommandBufferHelper(vk::CommandBufferHelper *commandBuffer);
+
   private:
     angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
@@ -452,6 +454,10 @@ class RendererVk : angle::NonCopyable
     std::mutex mCommandQueueMutex;
     vk::CommandQueue mCommandQueue;
 
+    // Command buffer pool management.
+    std::mutex mCommandBufferHelperFreeListMutex;
+    std::vector<vk::CommandBufferHelper *> mCommandBufferHelperFreeList;
+
     // Command Processor Thread
     vk::CommandProcessor mCommandProcessor;
     std::thread mCommandProcessorThread;

src/libANGLE/renderer/vulkan/SecondaryCommandBuffer.h

@@ -811,7 +811,7 @@ class SecondaryCommandBuffer final : angle::NonCopyable
         return writePointer + sizeInBytes;
     }
 
-    // flag to indicate that commandBuffer is open for new commands
+    // Flag to indicate that commandBuffer is open for new commands. Initially open.
     bool mIsOpen;
 
     std::vector<CommandHeader *> mCommands;
@@ -827,8 +827,9 @@ class SecondaryCommandBuffer final : angle::NonCopyable
 };
 
 ANGLE_INLINE SecondaryCommandBuffer::SecondaryCommandBuffer()
-    : mAllocator(nullptr), mCurrentWritePointer(nullptr), mCurrentBytesRemaining(0)
+    : mIsOpen(true), mAllocator(nullptr), mCurrentWritePointer(nullptr), mCurrentBytesRemaining(0)
 {}
+
 ANGLE_INLINE SecondaryCommandBuffer::~SecondaryCommandBuffer() {}
 
 // begin and insert DebugUtilsLabelEXT funcs share this same function body

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -1305,6 +1305,7 @@ angle::Result CommandBufferHelper::flushToPrimary(const angle::FeaturesVk &featu
     {
         mCommandBuffer.executeCommands(primary->getHandle());
     }
+
     // Restart the command buffer.
     reset();
 
@@ -1442,11 +1443,6 @@ void CommandBufferHelper::reset()
     ASSERT(mRenderPassUsedImages.empty());
 }
 
-void CommandBufferHelper::releaseToContextQueue(ContextVk *contextVk)
-{
-    contextVk->recycleCommandBuffer(this);
-}
-
 void CommandBufferHelper::resumeTransformFeedback()
 {
     ASSERT(mIsRenderPassCommandBuffer);

src/libANGLE/renderer/vulkan/vk_helpers.h

@@ -974,7 +974,6 @@ class CommandBufferHelper : angle::NonCopyable
 #endif
 
     void reset();
-    void releaseToContextQueue(ContextVk *contextVk);
 
     // Returns true if we have no work to execute. For renderpass command buffer, even if the
     // underlying command buffer is empty, we may still have a renderpass with an empty command