Vulkan: Move primary command buffer to CommandQueue.

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -382,6 +382,7 @@ CommandQueue::~CommandQueue() = default;
 
 void CommandQueue::destroy(VkDevice device)
 {
+    mPrimaryCommands.destroy(device);
     mPrimaryCommandPool.destroy(device);
     ASSERT(mInFlightCommands.empty() && mGarbageQueue.empty());
 }
@@ -604,15 +605,29 @@ angle::Result CommandQueue::finishToSerial(vk::Context *context,
     return retireFinishedCommands(context, finishedCount);
 }
 
-angle::Result CommandQueue::submitFrame(vk::Context *context,
-                                        egl::ContextPriority priority,
-                                        const VkSubmitInfo &submitInfo,
-                                        const vk::Shared<vk::Fence> &sharedFence,
-                                        vk::ResourceUseList *resourceList,
-                                        vk::GarbageList *currentGarbage,
-                                        vk::CommandPool *commandPool,
-                                        vk::PrimaryCommandBuffer &&commandBuffer)
+angle::Result CommandQueue::submitFrame(
+    vk::Context *context,
+    egl::ContextPriority priority,
+    const std::vector<VkSemaphore> &waitSemaphores,
+    const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
+    const vk::Semaphore *signalSemaphore,
+    const vk::Shared<vk::Fence> &sharedFence,
+    vk::ResourceUseList *resourceList,
+    vk::GarbageList *currentGarbage,
+    vk::CommandPool *commandPool)
 {
+    // Start an empty primary buffer if we have an empty submit.
+    if (!hasPrimaryCommands())
+    {
+        ANGLE_TRY(startPrimaryCommandBuffer(context));
+    }
+
+    ANGLE_VK_TRY(context, mPrimaryCommands.end());
+
+    VkSubmitInfo submitInfo = {};
+    InitializeSubmitInfo(&submitInfo, mPrimaryCommands, waitSemaphores, waitSemaphoreStageMasks,
+                         signalSemaphore);
+
     ANGLE_TRACE_EVENT0("gpu.angle", "CommandQueue::submitFrame");
     ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
 
@@ -633,7 +648,7 @@ angle::Result CommandQueue::submitFrame(vk::Context *context,
 
     // Store the primary CommandBuffer and command pool used for secondary CommandBuffers
     // in the in-flight list.
-    ANGLE_TRY(releaseToCommandBatch(context, std::move(commandBuffer), commandPool, &batch));
+    ANGLE_TRY(releaseToCommandBatch(context, std::move(mPrimaryCommands), commandPool, &batch));
 
     mInFlightCommands.emplace_back(scopedBatch.release());
 
@@ -663,6 +678,50 @@ vk::Shared<vk::Fence> CommandQueue::getLastSubmittedFence(const vk::Context *con
     return fence;
 }
 
+angle::Result CommandQueue::startPrimaryCommandBuffer(vk::Context *context)
+{
+    ASSERT(!mPrimaryCommands.valid());
+
+    ANGLE_TRY(allocatePrimaryCommandBuffer(context, &mPrimaryCommands));
+
+    VkCommandBufferBeginInfo beginInfo = {};
+    beginInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+    beginInfo.flags                    = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+    beginInfo.pInheritanceInfo         = nullptr;
+    ANGLE_VK_TRY(context, mPrimaryCommands.begin(beginInfo));
+
+    return angle::Result::Continue;
+}
+
+angle::Result CommandQueue::flushOutsideRPCommands(vk::Context *context,
+                                                   vk::CommandBufferHelper *outsideRPCommands)
+{
+    if (!mPrimaryCommands.valid())
+    {
+        ANGLE_TRY(startPrimaryCommandBuffer(context));
+    }
+
+    ANGLE_TRY(outsideRPCommands->flushToPrimary(context->getRenderer()->getFeatures(),
+                                                &mPrimaryCommands, nullptr));
+
+    return angle::Result::Continue;
+}
+
+angle::Result CommandQueue::flushRenderPassCommands(vk::Context *context,
+                                                    const vk::RenderPass &renderPass,
+                                                    vk::CommandBufferHelper *renderPassCommands)
+{
+    if (!mPrimaryCommands.valid())
+    {
+        ANGLE_TRY(startPrimaryCommandBuffer(context));
+    }
+
+    ANGLE_TRY(renderPassCommands->flushToPrimary(context->getRenderer()->getFeatures(),
+                                                 &mPrimaryCommands, &renderPass));
+
+    return angle::Result::Continue;
+}
+
 egl::ContextPriority GetContextPriority(const gl::State &state)
 {
     return egl::FromEGLenum<egl::ContextPriority>(state.getContextPriority());
@@ -698,7 +757,6 @@ ContextVk::ContextVk(const gl::State &state, gl::ErrorSet *errorSet, RendererVk
       mUseOldRewriteStructSamplers(false),
       mOutsideRenderPassCommands(nullptr),
       mRenderPassCommands(nullptr),
-      mHasPrimaryCommands(false),
       mGpuEventsEnabled(false),
       mSyncObjectPendingFlush(false),
       mDeferredFlushCount(0),
@@ -846,7 +904,6 @@ void ContextVk::onDestroy(const gl::Context *context)
     mShaderLibrary.destroy(device);
     mGpuEventQueryPool.destroy(device);
     mCommandPool.destroy(device);
-    mPrimaryCommands.destroy(device);
 
     // This will clean up any outstanding buffer allocations
     (void)mRenderer->cleanupGarbage(false);
@@ -958,8 +1015,6 @@ angle::Result ContextVk::initialize()
     getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
     getNextAvailableCommandBuffer(&mRenderPassCommands, true);
 
-    ANGLE_TRY(startPrimaryCommandBuffer());
-
     if (mGpuEventsEnabled)
     {
         // GPU events should only be available if timestamp queries are available.
@@ -1011,20 +1066,6 @@ angle::Result ContextVk::initialize()
     return angle::Result::Continue;
 }
 
-angle::Result ContextVk::startPrimaryCommandBuffer()
-{
-    ANGLE_TRY(mCommandQueue.allocatePrimaryCommandBuffer(this, &mPrimaryCommands));
-
-    VkCommandBufferBeginInfo beginInfo = {};
-    beginInfo.sType                    = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
-    beginInfo.flags                    = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
-    beginInfo.pInheritanceInfo         = nullptr;
-    ANGLE_VK_TRY(this, mPrimaryCommands.begin(beginInfo));
-
-    mHasPrimaryCommands = false;
-    return angle::Result::Continue;
-}
-
 angle::Result ContextVk::flush(const gl::Context *context)
 {
     // If we are in middle of renderpass and it is not a shared context, then we will defer the
@@ -1863,9 +1904,8 @@ void ContextVk::commandProcessorSyncErrorsAndQueueCommand(vk::CommandProcessorTa
     mRenderer->queueCommand(this, command);
 }
 
-angle::Result ContextVk::submitFrame(const VkSubmitInfo &submitInfo,
-                                     vk::ResourceUseList *resourceList,
-                                     vk::PrimaryCommandBuffer &&commandBuffer)
+angle::Result ContextVk::submitFrame(const vk::Semaphore *signalSemaphore,
+                                     vk::ResourceUseList *resourceList)
 {
     ASSERT(!getRenderer()->getFeatures().commandProcessor.enabled);
 
@@ -1875,9 +1915,9 @@ angle::Result ContextVk::submitFrame(const VkSubmitInfo &submitInfo,
     }
 
     ANGLE_TRY(ensureSubmitFenceInitialized());
-    ANGLE_TRY(mCommandQueue.submitFrame(this, mContextPriority, submitInfo, mSubmitFence,
-                                        resourceList, &mCurrentGarbage, &mCommandPool,
-                                        std::move(commandBuffer)));
+    ANGLE_TRY(mCommandQueue.submitFrame(this, mContextPriority, mWaitSemaphores,
+                                        mWaitSemaphoreStageMasks, signalSemaphore, mSubmitFence,
+                                        resourceList, &mCurrentGarbage, &mCommandPool));
 
     onRenderPassFinished();
     mComputeDirtyBits |= mNewComputeCommandBufferDirtyBits;
@@ -4377,7 +4417,7 @@ bool ContextVk::hasRecordedCommands()
 {
     ASSERT(mOutsideRenderPassCommands && mRenderPassCommands);
     return !mOutsideRenderPassCommands->empty() || mRenderPassCommands->started() ||
-           mHasPrimaryCommands;
+           mCommandQueue.hasPrimaryCommands();
 }
 
 angle::Result ContextVk::flushImpl(const vk::Semaphore *signalSemaphore)
@@ -4469,15 +4509,8 @@ angle::Result ContextVk::flushImpl(const vk::Semaphore *signalSemaphore)
     }
     else
     {
-        ANGLE_VK_TRY(this, mPrimaryCommands.end());
-
-        VkSubmitInfo submitInfo = {};
-        InitializeSubmitInfo(&submitInfo, mPrimaryCommands, mWaitSemaphores,
-                             mWaitSemaphoreStageMasks, signalSemaphore);
-
-        ANGLE_TRY(submitFrame(submitInfo, &mResourceUseList, std::move(mPrimaryCommands)));
+        ANGLE_TRY(submitFrame(signalSemaphore, &mResourceUseList));
 
-        ANGLE_TRY(startPrimaryCommandBuffer());
         mWaitSemaphores.clear();
         mWaitSemaphoreStageMasks.clear();
     }
@@ -5028,7 +5061,9 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
     }
 
     vk::RenderPass *renderPass = nullptr;
-    ANGLE_TRY(mRenderPassCommands->getRenderPassWithOps(this, &renderPass));
+    ANGLE_TRY(getRenderPassWithOps(mRenderPassCommands->getRenderPassDesc(),
+                                   mRenderPassCommands->getAttachmentOps(), &renderPass));
+
     if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         mRenderPassCommands->markClosed();
@@ -5040,12 +5075,9 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
     }
     else
     {
-        ANGLE_TRY(mRenderPassCommands->flushToPrimary(this->getFeatures(), &mPrimaryCommands,
-                                                      renderPass));
+        ANGLE_TRY(mCommandQueue.flushRenderPassCommands(this, *renderPass, mRenderPassCommands));
     }
 
-    mHasPrimaryCommands = true;
-
     if (mGpuEventsEnabled)
     {
         EventName eventName = GetTraceEventName("RP", mPerfCounters.renderPasses);
@@ -5177,23 +5209,19 @@ angle::Result ContextVk::flushOutsideRenderPassCommands()
         mOutsideRenderPassCommands->addCommandDiagnostics(this);
     }
 
-    vk::RenderPass *renderPass = nullptr;
-    ANGLE_TRY(mOutsideRenderPassCommands->getRenderPassWithOps(this, &renderPass));
     if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         mOutsideRenderPassCommands->markClosed();
         vk::CommandProcessorTask flushToPrimary;
-        flushToPrimary.initProcessCommands(this, mOutsideRenderPassCommands, renderPass);
+        flushToPrimary.initProcessCommands(this, mOutsideRenderPassCommands, nullptr);
         ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::flushOutsideRenderPassCommands");
         commandProcessorSyncErrorsAndQueueCommand(&flushToPrimary);
         getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
     }
     else
     {
-        ANGLE_TRY(mOutsideRenderPassCommands->flushToPrimary(getFeatures(), &mPrimaryCommands,
-                                                             renderPass));
+        ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(this, mOutsideRenderPassCommands));
     }
-    mHasPrimaryCommands = true;
     mPerfCounters.flushedOutsideRenderPassCommandBuffers++;
     return angle::Result::Continue;
 }

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -46,23 +46,19 @@ class CommandQueue final : angle::NonCopyable
 
     bool hasInFlightCommands() const;
 
-    angle::Result allocatePrimaryCommandBuffer(vk::Context *context,
-                                               vk::PrimaryCommandBuffer *commandBufferOut);
-    angle::Result releasePrimaryCommandBuffer(vk::Context *context,
-                                              vk::PrimaryCommandBuffer &&commandBuffer);
-
     void clearAllGarbage(RendererVk *renderer);
 
     angle::Result finishToSerial(vk::Context *context, Serial finishSerial, uint64_t timeout);
 
     angle::Result submitFrame(vk::Context *context,
                               egl::ContextPriority priority,
-                              const VkSubmitInfo &submitInfo,
+                              const std::vector<VkSemaphore> &waitSemaphores,
+                              const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
+                              const vk::Semaphore *signalSemaphore,
                               const vk::Shared<vk::Fence> &sharedFence,
                               vk::ResourceUseList *resourceList,
                               vk::GarbageList *currentGarbage,
-                              vk::CommandPool *commandPool,
-                              vk::PrimaryCommandBuffer &&commandBuffer);
+                              vk::CommandPool *commandPool);
 
     vk::Shared<vk::Fence> getLastSubmittedFence(const vk::Context *context) const;
 
@@ -71,17 +67,33 @@ class CommandQueue final : angle::NonCopyable
     // result). It would be nice if we didn't have to expose this for QueryVk::getResult.
     angle::Result checkCompletedCommands(vk::Context *context);
 
+    angle::Result flushOutsideRPCommands(vk::Context *context,
+                                         vk::CommandBufferHelper *outsideRPCommands);
+
+    angle::Result flushRenderPassCommands(vk::Context *context,
+                                          const vk::RenderPass &renderPass,
+                                          vk::CommandBufferHelper *renderPassCommands);
+
+    // TODO(jmadill): Remove this. b/172704839
+    bool hasPrimaryCommands() const { return mPrimaryCommands.valid(); }
+
   private:
     angle::Result releaseToCommandBatch(vk::Context *context,
                                         vk::PrimaryCommandBuffer &&commandBuffer,
                                         vk::CommandPool *commandPool,
                                         vk::CommandBatch *batch);
     angle::Result retireFinishedCommands(vk::Context *context, size_t finishedCount);
+    angle::Result startPrimaryCommandBuffer(vk::Context *context);
+    angle::Result allocatePrimaryCommandBuffer(vk::Context *context,
+                                               vk::PrimaryCommandBuffer *commandBufferOut);
+    angle::Result releasePrimaryCommandBuffer(vk::Context *context,
+                                              vk::PrimaryCommandBuffer &&commandBuffer);
 
     vk::GarbageQueue mGarbageQueue;
     std::vector<vk::CommandBatch> mInFlightCommands;
 
     // Keeps a free list of reusable primary command buffers.
+    vk::PrimaryCommandBuffer mPrimaryCommands;
     vk::PersistentCommandPool mPrimaryCommandPool;
 };
 
@@ -922,9 +934,8 @@ class ContextVk : public ContextImpl, public vk::Context
 
     void writeAtomicCounterBufferDriverUniformOffsets(uint32_t *offsetsOut, size_t offsetsSize);
 
-    angle::Result submitFrame(const VkSubmitInfo &submitInfo,
-                              vk::ResourceUseList *resourceList,
-                              vk::PrimaryCommandBuffer &&commandBuffer);
+    angle::Result submitFrame(const vk::Semaphore *signalSemaphore,
+                              vk::ResourceUseList *resourceList);
     angle::Result memoryBarrierImpl(GLbitfield barriers, VkPipelineStageFlags stageMask);
 
     angle::Result synchronizeCpuGpuTime();
@@ -939,7 +950,6 @@ class ContextVk : public ContextImpl, public vk::Context
     bool shouldUseOldRewriteStructSamplers() const;
     void clearAllGarbage();
     angle::Result ensureSubmitFenceInitialized();
-    angle::Result startPrimaryCommandBuffer();
     bool hasRecordedCommands();
     void dumpCommandStreamDiagnostics();
     angle::Result flushOutsideRenderPassCommands();
@@ -1117,9 +1127,6 @@ class ContextVk : public ContextImpl, public vk::Context
 
     vk::CommandBufferHelper *mOutsideRenderPassCommands;
     vk::CommandBufferHelper *mRenderPassCommands;
-    vk::PrimaryCommandBuffer mPrimaryCommands;
-    // Function recycleCommandBuffer() is public above
-    bool mHasPrimaryCommands;
 
     // Transform feedback buffers.
     angle::FastUnorderedSet<const vk::BufferHelper *,

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -1250,20 +1250,9 @@ void CommandBufferHelper::invalidateRenderPassStencilAttachment(
     ExtendRenderPassInvalidateArea(invalidateArea, &mStencilInvalidateArea);
 }
 
-angle::Result CommandBufferHelper::getRenderPassWithOps(ContextVk *contextVk,
-                                                        RenderPass **renderPass)
-{
-    *renderPass = nullptr;
-    if (mIsRenderPassCommandBuffer)
-    {
-        ANGLE_TRY(contextVk->getRenderPassWithOps(mRenderPassDesc, mAttachmentOps, renderPass));
-    }
-    return angle::Result::Continue;
-}
-
 angle::Result CommandBufferHelper::flushToPrimary(const angle::FeaturesVk &features,
                                                   PrimaryCommandBuffer *primary,
-                                                  RenderPass *renderPass)
+                                                  const RenderPass *renderPass)
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "CommandBufferHelper::flushToPrimary");
     ASSERT(!empty());

src/libANGLE/renderer/vulkan/vk_helpers.h

@@ -954,10 +954,9 @@ class CommandBufferHelper : angle::NonCopyable
 
     CommandBuffer &getCommandBuffer() { return mCommandBuffer; }
 
-    angle::Result getRenderPassWithOps(ContextVk *contextVk, RenderPass **renderPass);
     angle::Result flushToPrimary(const angle::FeaturesVk &features,
                                  PrimaryCommandBuffer *primary,
-                                 RenderPass *renderPass);
+                                 const RenderPass *renderPass);
 
     void executeBarriers(const angle::FeaturesVk &features, PrimaryCommandBuffer *primary);
 
@@ -1103,6 +1102,9 @@ class CommandBufferHelper : angle::NonCopyable
 
     void addCommandDiagnostics(ContextVk *contextVk);
 
+    const RenderPassDesc &getRenderPassDesc() const { return mRenderPassDesc; }
+    const AttachmentOpsArray &getAttachmentOps() const { return mAttachmentOps; }
+
   private:
     bool onDepthStencilAccess(ResourceAccess access,
                               uint32_t *cmdCountInvalidated,