Vulkan: Move vk::CommandQueue to RendererVk.

src/libANGLE/renderer/vulkan/CommandProcessor.cpp

@@ -114,7 +114,7 @@ void CommandProcessorTask::initTask()
 // CommandProcessorTask implementation
 void CommandProcessorTask::initProcessCommands(ContextVk *contextVk,
                                                CommandBufferHelper *commandBuffer,
-                                               RenderPass *renderPass)
+                                               const RenderPass *renderPass)
 {
     mTask          = CustomTask::ProcessCommands;
     mContextVk     = contextVk;
@@ -562,19 +562,6 @@ angle::Result TaskProcessor::submitFrame(Context *context,
     return angle::Result::Continue;
 }
 
-Shared<Fence> TaskProcessor::getLastSubmittedFenceWithLock(VkDevice device) const
-{
-    Shared<Fence> fence;
-    std::lock_guard<std::mutex> inFlightLock(mInFlightCommandsMutex);
-
-    if (!mInFlightCommands.empty())
-    {
-        fence.copy(device, mInFlightCommands.back().fence);
-    }
-
-    return fence;
-}
-
 angle::Result TaskProcessor::queueSubmit(Context *context,
                                          VkQueue queue,
                                          const VkSubmitInfo &submitInfo,
@@ -926,21 +913,6 @@ void CommandProcessor::shutdown(std::thread *commandProcessorThread)
     }
 }
 
-// Return the fence for the last submit. This may mean waiting on the worker to process tasks to
-// actually get to the last submit
-Shared<Fence> CommandProcessor::getLastSubmittedFence(const Context *context) const
-{
-    ANGLE_TRACE_EVENT0("gpu.angle", "CommandProcessor::getLastSubmittedFence");
-    std::unique_lock<std::mutex> lock(mWorkerMutex);
-    if (context->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled)
-    {
-        mWorkerIdleCondition.wait(lock, [this] { return (mTasks.empty() && mWorkerThreadIdle); });
-    }
-    // Worker thread is idle and command queue is empty so good to continue
-
-    return mTaskProcessor.getLastSubmittedFenceWithLock(getDevice());
-}
-
 Serial CommandProcessor::getLastSubmittedSerial()
 {
     std::lock_guard<std::mutex> lock(mCommandProcessorQueueSerialMutex);
@@ -1165,9 +1137,9 @@ void CommandQueue::handleDeviceLost(RendererVk *renderer)
     mInFlightCommands.clear();
 }
 
-bool CommandQueue::hasInFlightCommands() const
+bool CommandQueue::allInFlightCommandsAreAfterSerial(Serial serial) const
 {
-    return !mInFlightCommands.empty();
+    return mInFlightCommands.empty() || mInFlightCommands[0].serial > serial;
 }
 
 angle::Result CommandQueue::finishToSerial(Context *context, Serial finishSerial, uint64_t timeout)
@@ -1186,15 +1158,7 @@ angle::Result CommandQueue::finishToSerial(Context *context, Serial finishSerial
 
     size_t finishedCount = 0;
     while (finishedCount < mInFlightCommands.size() &&
-           mInFlightCommands[finishedCount].serial < finishSerial)
-    {
-        finishedCount++;
-    }
-
-    // This heuristic attempts to increase chances of success for shared resource scenarios.
-    // Ultimately as long as fences are managed by ContextVk there are edge case bugs here.
-    // TODO: http://anglebug.com/5217: fix this bug by moving it submit into RendererVk.
-    if (finishedCount < mInFlightCommands.size())
+           mInFlightCommands[finishedCount].serial <= finishSerial)
     {
         finishedCount++;
     }
@@ -1213,7 +1177,10 @@ angle::Result CommandQueue::finishToSerial(Context *context, Serial finishSerial
     ANGLE_VK_TRY(context, status);
 
     // Clean up finished batches.
-    return retireFinishedCommands(context, finishedCount);
+    ANGLE_TRY(retireFinishedCommands(context, finishedCount));
+    ASSERT(allInFlightCommandsAreAfterSerial(finishSerial));
+
+    return angle::Result::Continue;
 }
 
 angle::Result CommandQueue::submitFrame(
@@ -1222,9 +1189,9 @@ angle::Result CommandQueue::submitFrame(
     const std::vector<VkSemaphore> &waitSemaphores,
     const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
     const Semaphore *signalSemaphore,
-    const Shared<Fence> &sharedFence,
-    ResourceUseList *resourceList,
-    GarbageList *currentGarbage,
+    Shared<Fence> &&sharedFence,
+    ResourceUseList &&resourceList,
+    GarbageList &&currentGarbage,
     CommandPool *commandPool)
 {
     // Start an empty primary buffer if we have an empty submit.
@@ -1243,14 +1210,14 @@ angle::Result CommandQueue::submitFrame(
 
     DeviceScoped<CommandBatch> scopedBatch(device);
     CommandBatch &batch = scopedBatch.get();
-    batch.fence.copy(device, sharedFence);
+    batch.fence         = std::move(sharedFence);
 
-    ANGLE_TRY(renderer->queueSubmit(context, priority, submitInfo, resourceList, &batch.fence.get(),
-                                    &batch.serial));
+    ANGLE_TRY(renderer->queueSubmit(context, priority, submitInfo, std::move(resourceList),
+                                    &batch.fence.get(), &batch.serial));
 
-    if (!currentGarbage->empty())
+    if (!currentGarbage.empty())
     {
-        mGarbageQueue.emplace_back(std::move(*currentGarbage), batch.serial);
+        mGarbageQueue.emplace_back(std::move(currentGarbage), batch.serial);
     }
 
     // Store the primary CommandBuffer and command pool used for secondary CommandBuffers
@@ -1272,17 +1239,52 @@ angle::Result CommandQueue::submitFrame(
     return angle::Result::Continue;
 }
 
-Shared<Fence> CommandQueue::getLastSubmittedFence(const Context *context) const
+angle::Result CommandQueue::waitForSerialWithUserTimeout(vk::Context *context,
+                                                         Serial serial,
+                                                         uint64_t timeout,
+                                                         VkResult *result)
 {
-    ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
+    // No in-flight work. This indicates the serial is already complete.
+    if (mInFlightCommands.empty())
+    {
+        *result = VK_SUCCESS;
+        return angle::Result::Continue;
+    }
 
-    Shared<Fence> fence;
-    if (!mInFlightCommands.empty())
+    // Serial is already complete.
+    if (serial < mInFlightCommands[0].serial)
     {
-        fence.copy(context->getDevice(), mInFlightCommands.back().fence);
+        *result = VK_SUCCESS;
+        return angle::Result::Continue;
     }
 
-    return fence;
+    size_t batchIndex = 0;
+    while (batchIndex != mInFlightCommands.size() && mInFlightCommands[batchIndex].serial < serial)
+    {
+        batchIndex++;
+    }
+
+    // Serial is not yet submitted. This is undefined behaviour, so we can do anything.
+    if (batchIndex >= mInFlightCommands.size())
+    {
+        WARN() << "Waiting on an unsubmitted serial.";
+        *result = VK_TIMEOUT;
+        return angle::Result::Continue;
+    }
+
+    ASSERT(serial == mInFlightCommands[batchIndex].serial);
+
+    vk::Fence &fence = mInFlightCommands[batchIndex].fence.get();
+    ASSERT(fence.valid());
+    *result = fence.wait(context->getDevice(), timeout);
+
+    // Don't trigger an error on timeout.
+    if (*result != VK_TIMEOUT)
+    {
+        ANGLE_VK_TRY(context, *result);
+    }
+
+    return angle::Result::Continue;
 }
 
 angle::Result CommandQueue::ensurePrimaryCommandBufferValid(Context *context)

src/libANGLE/renderer/vulkan/CommandProcessor.h

@@ -65,7 +65,7 @@ class CommandProcessorTask
 
     void initProcessCommands(ContextVk *contextVk,
                              CommandBufferHelper *commandBuffer,
-                             RenderPass *renderPass);
+                             const RenderPass *renderPass);
 
     void initPresent(egl::ContextPriority priority, VkPresentInfoKHR &presentInfo);
 
@@ -104,7 +104,7 @@ class CommandProcessorTask
     const VkCommandBuffer &getOneOffCommandBufferVk() const { return mOneOffCommandBufferVk; }
     const Fence *getOneOffFence() { return mOneOffFence; }
     const VkPresentInfoKHR &getPresentInfo() const { return mPresentInfo; }
-    RenderPass *getRenderPass() const { return mRenderPass; }
+    const RenderPass *getRenderPass() const { return mRenderPass; }
     CommandBufferHelper *getCommandBuffer() const { return mCommandBuffer; }
     ContextVk *getContextVk() const { return mContextVk; }
 
@@ -115,7 +115,7 @@ class CommandProcessorTask
 
     // ProcessCommands
     ContextVk *mContextVk;
-    RenderPass *mRenderPass;
+    const RenderPass *mRenderPass;
     CommandBufferHelper *mCommandBuffer;
 
     // Flush data
@@ -172,8 +172,6 @@ class CommandQueue final : angle::NonCopyable
     void destroy(VkDevice device);
     void handleDeviceLost(RendererVk *renderer);
 
-    bool hasInFlightCommands() const;
-
     void clearAllGarbage(RendererVk *renderer);
 
     angle::Result finishToSerial(Context *context, Serial finishSerial, uint64_t timeout);
@@ -183,12 +181,15 @@ class CommandQueue final : angle::NonCopyable
                               const std::vector<VkSemaphore> &waitSemaphores,
                               const std::vector<VkPipelineStageFlags> &waitSemaphoreStageMasks,
                               const Semaphore *signalSemaphore,
-                              const Shared<Fence> &sharedFence,
-                              ResourceUseList *resourceList,
-                              GarbageList *currentGarbage,
+                              Shared<Fence> &&sharedFence,
+                              ResourceUseList &&resourceList,
+                              GarbageList &&currentGarbage,
                               CommandPool *commandPool);
 
-    Shared<Fence> getLastSubmittedFence(const Context *context) const;
+    angle::Result waitForSerialWithUserTimeout(vk::Context *context,
+                                               Serial serial,
+                                               uint64_t timeout,
+                                               VkResult *result);
 
     // Check to see which batches have finished completion (forward progress for
     // mLastCompletedQueueSerial, for example for when the application busy waits on a query
@@ -213,6 +214,8 @@ class CommandQueue final : angle::NonCopyable
     angle::Result releasePrimaryCommandBuffer(Context *context,
                                               PrimaryCommandBuffer &&commandBuffer);
 
+    bool allInFlightCommandsAreAfterSerial(Serial serial) const;
+
     GarbageQueue mGarbageQueue;
     std::vector<CommandBatch> mInFlightCommands;
 
@@ -252,8 +255,6 @@ class TaskProcessor : angle::NonCopyable
                               const VkSubmitInfo &submitInfo,
                               const Fence *fence);
 
-    Shared<Fence> getLastSubmittedFenceWithLock(VkDevice device) const;
-
     void handleDeviceLost(Context *context);
 
     // Called by CommandProcessor to process any completed work
@@ -320,7 +321,6 @@ class CommandProcessor : public Context
     // Wait until desired serial has been processed.
     void finishToSerial(Context *context, Serial serial);
 
-    Shared<Fence> getLastSubmittedFence(const Context *context) const;
     void handleDeviceLost();
 
     bool hasPendingError() const

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -507,14 +507,11 @@ void ContextVk::onDestroy(const gl::Context *context)
 
     ASSERT(mCurrentGarbage.empty());
 
-    mCommandQueue.destroy(device);
-
     mRenderer->releaseSharedResources(&mResourceUseList);
 
     mUtils.destroy(mRenderer);
 
     mRenderPassCache.destroy(device);
-    mSubmitFence.reset(device);
     mShaderLibrary.destroy(device);
     mGpuEventQueryPool.destroy(device);
     mCommandPool.destroy(device);
@@ -600,8 +597,6 @@ angle::Result ContextVk::initialize()
         buffer.init(mRenderer, kVertexBufferUsage, 1, kDefaultBufferSize, true);
     }
 
-    ANGLE_TRY(mCommandQueue.init(this));
-
 #if ANGLE_ENABLE_VULKAN_GPU_TRACE_EVENTS
     angle::PlatformMethods *platform = ANGLEPlatformCurrent();
     ASSERT(platform);
@@ -1500,24 +1495,6 @@ void ContextVk::addOverlayUsedBuffersCount(vk::CommandBufferHelper *commandBuffe
     }
 }
 
-void ContextVk::commandProcessorSyncErrors()
-{
-    while (mRenderer->hasPendingError())
-    {
-        vk::Error error = mRenderer->getAndClearPendingError();
-        if (error.mErrorCode != VK_SUCCESS)
-        {
-            handleError(error.mErrorCode, error.mFile, error.mFunction, error.mLine);
-        }
-    }
-}
-
-void ContextVk::commandProcessorSyncErrorsAndQueueCommand(vk::CommandProcessorTask *command)
-{
-    commandProcessorSyncErrors();
-    mRenderer->queueCommand(this, command);
-}
-
 angle::Result ContextVk::submitFrame(const vk::Semaphore *signalSemaphore)
 {
     if (mCurrentWindowSurface)
@@ -1536,27 +1513,9 @@ angle::Result ContextVk::submitFrame(const vk::Semaphore *signalSemaphore)
         dumpCommandStreamDiagnostics();
     }
 
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        vk::CommandProcessorTask flushAndQueueSubmit;
-        flushAndQueueSubmit.initFlushAndQueueSubmit(
-            std::move(mWaitSemaphores), std::move(mWaitSemaphoreStageMasks), signalSemaphore,
-            mContextPriority, std::move(mCurrentGarbage), std::move(mResourceUseList));
-
-        commandProcessorSyncErrorsAndQueueCommand(&flushAndQueueSubmit);
-    }
-    else
-    {
-        ANGLE_TRY(ensureSubmitFenceInitialized());
-        ANGLE_TRY(mCommandQueue.submitFrame(this, mContextPriority, mWaitSemaphores,
-                                            mWaitSemaphoreStageMasks, signalSemaphore, mSubmitFence,
-                                            &mResourceUseList, &mCurrentGarbage, &mCommandPool));
-        // Make sure a new fence is created for the next submission.
-        mRenderer->resetSharedFence(&mSubmitFence);
-
-        mWaitSemaphores.clear();
-        mWaitSemaphoreStageMasks.clear();
-    }
+    ANGLE_TRY(mRenderer->submitFrame(
+        this, mContextPriority, std::move(mWaitSemaphores), std::move(mWaitSemaphoreStageMasks),
+        signalSemaphore, std::move(mResourceUseList), std::move(mCurrentGarbage), &mCommandPool));
 
     onRenderPassFinished();
     mComputeDirtyBits |= mNewComputeCommandBufferDirtyBits;
@@ -1885,31 +1844,14 @@ void ContextVk::clearAllGarbage()
         garbage.destroy(mRenderer);
     }
     mCurrentGarbage.clear();
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        // Issue command to CommandProcessor to ensure all work is complete, which will return any
-        // garbage items as well.
-        mRenderer->finishAllWork(this);
-    }
-    else
-    {
-        mCommandQueue.clearAllGarbage(mRenderer);
-    }
+    mRenderer->clearAllGarbage(this);
 }
 
 void ContextVk::handleDeviceLost()
 {
     mOutsideRenderPassCommands->reset();
     mRenderPassCommands->reset();
-
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        mRenderer->handleDeviceLost();
-    }
-    else
-    {
-        mCommandQueue.handleDeviceLost(mRenderer);
-    }
+    mRenderer->handleDeviceLost();
     clearAllGarbage();
 
     mRenderer->notifyDeviceLost();
@@ -4136,16 +4078,7 @@ angle::Result ContextVk::finishImpl()
     ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::finishImpl");
 
     ANGLE_TRY(flushImpl(nullptr));
-
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        ANGLE_TRY(finishToSerial(getLastSubmittedQueueSerial()));
-    }
-    else
-    {
-        ANGLE_TRY(finishToSerial(getLastSubmittedQueueSerial()));
-        ASSERT(!mCommandQueue.hasInFlightCommands());
-    }
+    ANGLE_TRY(mRenderer->finish(this));
 
     clearAllGarbage();
 
@@ -4186,26 +4119,12 @@ bool ContextVk::isSerialInUse(Serial serial) const
 
 angle::Result ContextVk::checkCompletedCommands()
 {
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        // TODO: https://issuetracker.google.com/169788986 - would be better if we could just wait
-        // for the work we need but that requires QueryHelper to use the actual serial for the
-        // query.
-        mRenderer->checkCompletedCommands(this);
-        return angle::Result::Continue;
-    }
-
-    return mCommandQueue.checkCompletedCommands(this);
+    return mRenderer->checkCompletedCommands(this);
 }
 
 angle::Result ContextVk::finishToSerial(Serial serial)
 {
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        mRenderer->finishToSerial(this, serial);
-        return angle::Result::Continue;
-    }
-    return mCommandQueue.finishToSerial(this, serial, mRenderer->getMaxFenceWaitTimeNs());
+    return mRenderer->finishToSerial(this, serial);
 }
 
 angle::Result ContextVk::getCompatibleRenderPass(const vk::RenderPassDesc &desc,
@@ -4223,35 +4142,6 @@ angle::Result ContextVk::getRenderPassWithOps(const vk::RenderPassDesc &desc,
     return mRenderPassCache.getRenderPassWithOps(this, desc, ops, renderPassOut);
 }
 
-angle::Result ContextVk::ensureSubmitFenceInitialized()
-{
-    if (mSubmitFence.isReferenced())
-    {
-        return angle::Result::Continue;
-    }
-    return mRenderer->newSharedFence(this, &mSubmitFence);
-}
-
-angle::Result ContextVk::getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOut)
-{
-    ASSERT(!getRenderer()->getFeatures().commandProcessor.enabled);
-
-    ANGLE_TRY(ensureSubmitFenceInitialized());
-
-    ASSERT(!sharedFenceOut->isReferenced());
-    sharedFenceOut->copy(getDevice(), mSubmitFence);
-    return angle::Result::Continue;
-}
-
-vk::Shared<vk::Fence> ContextVk::getLastSubmittedFence() const
-{
-    if (mRenderer->getFeatures().commandProcessor.enabled)
-    {
-        return mRenderer->getLastSubmittedFence(this);
-    }
-    return mCommandQueue.getLastSubmittedFence(this);
-}
-
 angle::Result ContextVk::getTimestamp(uint64_t *timestampOut)
 {
     // The intent of this function is to query the timestamp without stalling the GPU.
@@ -4646,19 +4536,13 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
     ANGLE_TRY(getRenderPassWithOps(mRenderPassCommands->getRenderPassDesc(),
                                    mRenderPassCommands->getAttachmentOps(), &renderPass));
 
-    if (mRenderer->getFeatures().commandProcessor.enabled)
+    ANGLE_TRY(mRenderer->flushRenderPassCommands(this, *renderPass, &mRenderPassCommands));
+
+    // TODO(jmadill): Manage in RendererVk. b/172678125
+    if (getFeatures().commandProcessor.enabled)
     {
-        mRenderPassCommands->markClosed();
-        vk::CommandProcessorTask flushToPrimary;
-        flushToPrimary.initProcessCommands(this, mRenderPassCommands, renderPass);
-        ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::flushInsideRenderPassCommands");
-        commandProcessorSyncErrorsAndQueueCommand(&flushToPrimary);
         getNextAvailableCommandBuffer(&mRenderPassCommands, true);
     }
-    else
-    {
-        ANGLE_TRY(mCommandQueue.flushRenderPassCommands(this, *renderPass, mRenderPassCommands));
-    }
 
     if (mGpuEventsEnabled)
     {
@@ -4791,19 +4675,14 @@ angle::Result ContextVk::flushOutsideRenderPassCommands()
         mOutsideRenderPassCommands->addCommandDiagnostics(this);
     }
 
-    if (mRenderer->getFeatures().commandProcessor.enabled)
+    ANGLE_TRY(mRenderer->flushOutsideRPCommands(this, &mOutsideRenderPassCommands));
+
+    // TODO(jmadill): Manage in RendererVk. b/172678125
+    if (getFeatures().commandProcessor.enabled)
     {
-        mOutsideRenderPassCommands->markClosed();
-        vk::CommandProcessorTask flushToPrimary;
-        flushToPrimary.initProcessCommands(this, mOutsideRenderPassCommands, nullptr);
-        ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::flushOutsideRenderPassCommands");
-        commandProcessorSyncErrorsAndQueueCommand(&flushToPrimary);
         getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
     }
-    else
-    {
-        ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(this, mOutsideRenderPassCommands));
-    }
+
     mPerfCounters.flushedOutsideRenderPassCommandBuffers++;
     return angle::Result::Continue;
 }

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -386,10 +386,6 @@ class ContextVk : public ContextImpl, public vk::Context
                                        const vk::AttachmentOpsArray &ops,
                                        vk::RenderPass **renderPassOut);
 
-    // Get (or allocate) the fence that will be signaled on next submission.
-    angle::Result getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOut);
-    vk::Shared<vk::Fence> getLastSubmittedFence() const;
-
     vk::ShaderLibrary &getShaderLibrary() { return mShaderLibrary; }
     UtilsVk &getUtils() { return mUtils; }
 
@@ -522,6 +518,7 @@ class ContextVk : public ContextImpl, public vk::Context
 
     vk::CommandBuffer &getOutsideRenderPassCommandBuffer()
     {
+        ASSERT(!mOutsideRenderPassCommands->hasRenderPass());
         return mOutsideRenderPassCommands->getCommandBuffer();
     }
 
@@ -595,10 +592,6 @@ class ContextVk : public ContextImpl, public vk::Context
     void updateOverlayOnPresent();
     void addOverlayUsedBuffersCount(vk::CommandBufferHelper *commandBuffer);
 
-    // Sync any errors from the command processor
-    void commandProcessorSyncErrors();
-    // Sync any error from worker thread and queue up next command for processing
-    void commandProcessorSyncErrorsAndQueueCommand(vk::CommandProcessorTask *command);
     // When worker thread completes, it releases command buffers back to context queue
     void recycleCommandBuffer(vk::CommandBufferHelper *commandBuffer);
 
@@ -884,7 +877,6 @@ class ContextVk : public ContextImpl, public vk::Context
     bool shouldEmulateSeamfulCubeMapSampling() const;
     bool shouldUseOldRewriteStructSamplers() const;
     void clearAllGarbage();
-    angle::Result ensureSubmitFenceInitialized();
     bool hasRecordedCommands();
     void dumpCommandStreamDiagnostics();
     angle::Result flushOutsideRenderPassCommands();
@@ -1035,20 +1027,10 @@ class ContextVk : public ContextImpl, public vk::Context
     // We use a single pool for recording commands. We also keep a free list for pool recycling.
     vk::CommandPool mCommandPool;
 
-    // TODO: This can be killed once threading is enabled https://issuetracker.google.com/153666475
-    vk::CommandQueue mCommandQueue;
     vk::GarbageList mCurrentGarbage;
 
     RenderPassCache mRenderPassCache;
 
-    // mSubmitFence is the fence that's going to be signaled at the next submission.  This is used
-    // to support SyncVk objects, which may outlive the context (as EGLSync objects).
-    //
-    // TODO(geofflang): this is in preparation for moving RendererVk functionality to ContextVk, and
-    // is otherwise unnecessary as the SyncVk objects don't actually outlive the renderer currently.
-    // http://anglebug.com/2701
-    vk::Shared<vk::Fence> mSubmitFence;
-
     // We have a queue of CommandBufferHelpers (CBHs) that is drawn from for the two active command
     //  buffers in the main thread. The two active command buffers are the inside and outside
     //  RenderPass command buffers.

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -511,6 +511,7 @@ void RendererVk::onDestroy()
         // Shutdown worker thread
         mCommandProcessor.shutdown(&mCommandProcessorThread);
     }
+    mCommandQueue.destroy(mDevice);
 
     // Force all commands to finish by flushing all queues.
     for (VkQueue queue : mQueues)
@@ -929,6 +930,10 @@ angle::Result RendererVk::initialize(DisplayVk *displayVk,
             ANGLE_TRY(mCommandProcessor.initTaskProcessor(displayVk));
         }
     }
+    else
+    {
+        ANGLE_TRY(mCommandQueue.init(displayVk));
+    }
 
     return angle::Result::Continue;
 }
@@ -2240,7 +2245,7 @@ void RendererVk::outputVmaStatString()
 angle::Result RendererVk::queueSubmit(vk::Context *context,
                                       egl::ContextPriority priority,
                                       const VkSubmitInfo &submitInfo,
-                                      vk::ResourceUseList *resourceList,
+                                      vk::ResourceUseList &&resourceUseList,
                                       const vk::Fence *fence,
                                       Serial *serialOut)
 {
@@ -2257,10 +2262,8 @@ angle::Result RendererVk::queueSubmit(vk::Context *context,
         VkFence handle = fence ? fence->getHandle() : VK_NULL_HANDLE;
         ANGLE_VK_TRY(context, vkQueueSubmit(mQueues[priority], 1, &submitInfo, handle));
 
-        if (resourceList)
-        {
-            resourceList->releaseResourceUsesAndUpdateSerials(mCurrentQueueSerial);
-        }
+        resourceUseList.releaseResourceUsesAndUpdateSerials(mCurrentQueueSerial);
+
         *serialOut                = mCurrentQueueSerial;
         mLastSubmittedQueueSerial = mCurrentQueueSerial;
         mCurrentQueueSerial       = mQueueSerialFactory.generate();
@@ -2294,13 +2297,13 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
     }
     else
     {
-
         VkSubmitInfo submitInfo       = {};
         submitInfo.sType              = VK_STRUCTURE_TYPE_SUBMIT_INFO;
         submitInfo.commandBufferCount = 1;
         submitInfo.pCommandBuffers    = primary.ptr();
 
-        ANGLE_TRY(queueSubmit(context, priority, submitInfo, nullptr, fence, serialOut));
+        ANGLE_TRY(
+            queueSubmit(context, priority, submitInfo, vk::ResourceUseList(), fence, serialOut));
     }
 
     mPendingOneOffCommands.push_back({*serialOut, std::move(primary)});
@@ -2586,4 +2589,188 @@ angle::Result RendererVk::getCommandBufferOneOff(vk::Context *context,
 
     return angle::Result::Continue;
 }
+
+void RendererVk::commandProcessorSyncErrors(vk::Context *context)
+{
+    while (hasPendingError())
+    {
+        vk::Error error = getAndClearPendingError();
+        if (error.mErrorCode != VK_SUCCESS)
+        {
+            context->handleError(error.mErrorCode, error.mFile, error.mFunction, error.mLine);
+        }
+    }
+}
+
+void RendererVk::commandProcessorSyncErrorsAndQueueCommand(vk::Context *context,
+                                                           vk::CommandProcessorTask *command)
+{
+    commandProcessorSyncErrors(context);
+    queueCommand(context, command);
+}
+
+angle::Result RendererVk::submitFrame(vk::Context *context,
+                                      egl::ContextPriority contextPriority,
+                                      std::vector<VkSemaphore> &&waitSemaphores,
+                                      std::vector<VkPipelineStageFlags> &&waitSemaphoreStageMasks,
+                                      const vk::Semaphore *signalSemaphore,
+                                      vk::ResourceUseList &&resourceUseList,
+                                      vk::GarbageList &&currentGarbage,
+                                      vk::CommandPool *commandPool)
+{
+    if (mFeatures.commandProcessor.enabled)
+    {
+        vk::CommandProcessorTask flushAndQueueSubmit;
+        flushAndQueueSubmit.initFlushAndQueueSubmit(
+            std::move(waitSemaphores), std::move(waitSemaphoreStageMasks), signalSemaphore,
+            contextPriority, std::move(currentGarbage), std::move(resourceUseList));
+
+        commandProcessorSyncErrorsAndQueueCommand(context, &flushAndQueueSubmit);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        vk::Shared<vk::Fence> submitFence;
+        ANGLE_TRY(newSharedFence(context, &submitFence));
+        ANGLE_TRY(mCommandQueue.submitFrame(context, contextPriority, waitSemaphores,
+                                            waitSemaphoreStageMasks, signalSemaphore,
+                                            std::move(submitFence), std::move(resourceUseList),
+                                            std::move(currentGarbage), commandPool));
+
+        waitSemaphores.clear();
+        waitSemaphoreStageMasks.clear();
+    }
+
+    return angle::Result::Continue;
+}
+
+void RendererVk::clearAllGarbage(vk::Context *context)
+{
+    if (mFeatures.commandProcessor.enabled)
+    {
+        // Issue command to CommandProcessor to ensure all work is complete, which will return any
+        // garbage items as well.
+        finishAllWork(context);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        mCommandQueue.clearAllGarbage(this);
+    }
+}
+
+void RendererVk::handleDeviceLost()
+{
+    if (mFeatures.commandProcessor.enabled)
+    {
+        mCommandProcessor.handleDeviceLost();
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        mCommandQueue.handleDeviceLost(this);
+    }
+}
+
+angle::Result RendererVk::finishToSerial(vk::Context *context, Serial serial)
+{
+    if (mFeatures.commandProcessor.enabled)
+    {
+        mCommandProcessor.finishToSerial(context, serial);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        ANGLE_TRY(mCommandQueue.finishToSerial(context, serial, getMaxFenceWaitTimeNs()));
+    }
+
+    return angle::Result::Continue;
+}
+
+angle::Result RendererVk::waitForSerialWithUserTimeout(vk::Context *context,
+                                                       Serial serial,
+                                                       uint64_t timeout,
+                                                       VkResult *result)
+{
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::waitForSerialWithUserTimeout");
+    if (mFeatures.commandProcessor.enabled)
+    {
+        // TODO: https://issuetracker.google.com/170312581 - Wait with timeout.
+        mCommandProcessor.finishToSerial(context, serial);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        ANGLE_TRY(mCommandQueue.waitForSerialWithUserTimeout(context, serial, timeout, result));
+    }
+
+    return angle::Result::Continue;
+}
+
+angle::Result RendererVk::finish(vk::Context *context)
+{
+    ANGLE_TRY(finishToSerial(context, mLastSubmittedQueueSerial));
+    return angle::Result::Continue;
+}
+
+angle::Result RendererVk::checkCompletedCommands(vk::Context *context)
+{
+    if (mFeatures.commandProcessor.enabled)
+    {
+        // TODO: https://issuetracker.google.com/169788986 - would be better if we could just wait
+        // for the work we need but that requires QueryHelper to use the actual serial for the
+        // query.
+        mCommandProcessor.checkCompletedCommands(context);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        ANGLE_TRY(mCommandQueue.checkCompletedCommands(context));
+    }
+
+    return angle::Result::Continue;
+}
+
+angle::Result RendererVk::flushRenderPassCommands(ContextVk *contextVk,
+                                                  const vk::RenderPass &renderPass,
+                                                  vk::CommandBufferHelper **renderPassCommands)
+{
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::flushRenderPassCommands");
+    if (mFeatures.commandProcessor.enabled)
+    {
+        (*renderPassCommands)->markClosed();
+        vk::CommandProcessorTask flushToPrimary;
+        flushToPrimary.initProcessCommands(contextVk, *renderPassCommands, &renderPass);
+        commandProcessorSyncErrorsAndQueueCommand(contextVk, &flushToPrimary);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        ANGLE_TRY(
+            mCommandQueue.flushRenderPassCommands(contextVk, renderPass, *renderPassCommands));
+    }
+
+    return angle::Result::Continue;
+}
+
+angle::Result RendererVk::flushOutsideRPCommands(ContextVk *contextVk,
+                                                 vk::CommandBufferHelper **outsideRPCommands)
+{
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::flushOutsideRPCommands");
+    if (mFeatures.commandProcessor.enabled)
+    {
+        (*outsideRPCommands)->markClosed();
+        vk::CommandProcessorTask flushToPrimary;
+        flushToPrimary.initProcessCommands(contextVk, *outsideRPCommands, nullptr);
+        commandProcessorSyncErrorsAndQueueCommand(contextVk, &flushToPrimary);
+    }
+    else
+    {
+        std::lock_guard<std::mutex> lock(mCommandQueueMutex);
+        ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(contextVk, *outsideRPCommands));
+    }
+
+    return angle::Result::Continue;
+}
+
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -175,7 +175,7 @@ class RendererVk : angle::NonCopyable
     angle::Result queueSubmit(vk::Context *context,
                               egl::ContextPriority priority,
                               const VkSubmitInfo &submitInfo,
-                              vk::ResourceUseList *resourceList,
+                              vk::ResourceUseList &&resourceList,
                               const vk::Fence *fence,
                               Serial *serialOut);
     angle::Result queueWaitIdle(vk::Context *context, egl::ContextPriority priority);
@@ -230,12 +230,6 @@ class RendererVk : angle::NonCopyable
         }
     }
 
-    vk::Shared<vk::Fence> getLastSubmittedFence(const vk::Context *context) const
-    {
-        return mCommandProcessor.getLastSubmittedFence(context);
-    }
-    void handleDeviceLost() { mCommandProcessor.handleDeviceLost(); }
-
     angle::Result getPipelineCache(vk::PipelineCache **pipelineCache);
     void onNewGraphicsPipeline()
     {
@@ -298,16 +292,6 @@ class RendererVk : angle::NonCopyable
         mCommandProcessor.waitForWorkComplete(context);
     }
 
-    void finishToSerial(vk::Context *context, Serial serial)
-    {
-        mCommandProcessor.finishToSerial(context, serial);
-    }
-
-    void checkCompletedCommands(vk::Context *context)
-    {
-        mCommandProcessor.checkCompletedCommands(context);
-    }
-
     void finishAllWork(vk::Context *context) { mCommandProcessor.finishAllWork(context); }
     VkQueue getVkQueue(egl::ContextPriority priority) const { return mQueues[priority]; }
 
@@ -321,6 +305,32 @@ class RendererVk : angle::NonCopyable
 
     angle::Result cleanupGarbage(bool block);
 
+    angle::Result submitFrame(vk::Context *context,
+                              egl::ContextPriority contextPriority,
+                              std::vector<VkSemaphore> &&waitSemaphores,
+                              std::vector<VkPipelineStageFlags> &&waitSemaphoreStageMasks,
+                              const vk::Semaphore *signalSemaphore,
+                              vk::ResourceUseList &&resourceUseList,
+                              vk::GarbageList &&currentGarbage,
+                              vk::CommandPool *commandPool);
+
+    void clearAllGarbage(vk::Context *context);
+    void handleDeviceLost();
+    angle::Result finishToSerial(vk::Context *context, Serial serial);
+    angle::Result waitForSerialWithUserTimeout(vk::Context *context,
+                                               Serial serial,
+                                               uint64_t timeout,
+                                               VkResult *result);
+    angle::Result finish(vk::Context *context);
+    angle::Result checkCompletedCommands(vk::Context *context);
+
+    // TODO(jmadill): Use vk::Context instead of ContextVk. b/172704839
+    angle::Result flushRenderPassCommands(ContextVk *contextVk,
+                                          const vk::RenderPass &renderPass,
+                                          vk::CommandBufferHelper **renderPassCommands);
+    angle::Result flushOutsideRPCommands(ContextVk *contextVk,
+                                         vk::CommandBufferHelper **outsideRPCommands);
+
   private:
     angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
@@ -340,6 +350,12 @@ class RendererVk : angle::NonCopyable
     template <VkFormatFeatureFlags VkFormatProperties::*features>
     bool hasFormatFeatureBits(VkFormat format, const VkFormatFeatureFlags featureBits) const;
 
+    // Sync any errors from the command processor
+    void commandProcessorSyncErrors(vk::Context *context);
+    // Sync any error from worker thread and queue up next command for processing
+    void commandProcessorSyncErrorsAndQueueCommand(vk::Context *context,
+                                                   vk::CommandProcessorTask *command);
+
     egl::Display *mDisplay;
 
     mutable bool mCapsInitialized;
@@ -434,6 +450,9 @@ class RendererVk : angle::NonCopyable
     };
     std::deque<PendingOneOffCommands> mPendingOneOffCommands;
 
+    std::mutex mCommandQueueMutex;
+    vk::CommandQueue mCommandQueue;
+
     // Command Processor Thread
     vk::CommandProcessor mCommandProcessor;
     std::thread mCommandProcessorThread;

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -482,25 +482,6 @@ SwapchainImage::SwapchainImage(SwapchainImage &&other)
       presentHistory(std::move(other.presentHistory)),
       currentPresentHistoryIndex(other.currentPresentHistoryIndex)
 {}
-
-SwapHistory::SwapHistory() = default;
-
-SwapHistory::~SwapHistory() = default;
-
-void SwapHistory::destroy(RendererVk *renderer)
-{
-    renderer->resetSharedFence(&sharedFence);
-}
-
-angle::Result SwapHistory::waitFence(ContextVk *contextVk)
-{
-    ASSERT(sharedFence.isReferenced());
-    // TODO: https://issuetracker.google.com/170312581 - This wait needs to be synchronized with
-    // worker thread
-    ANGLE_VK_TRY(contextVk, sharedFence.get().wait(contextVk->getDevice(),
-                                                   std::numeric_limits<uint64_t>::max()));
-    return angle::Result::Continue;
-}
 }  // namespace impl
 
 using namespace impl;
@@ -550,11 +531,6 @@ void WindowSurfaceVk::destroy(const egl::Display *display)
 
     destroySwapChainImages(displayVk);
 
-    for (SwapHistory &swap : mSwapHistory)
-    {
-        swap.destroy(renderer);
-    }
-
     if (mSwapchain)
     {
         vkDestroySwapchainKHR(device, mSwapchain, nullptr);
@@ -1296,16 +1272,10 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
     RendererVk *renderer = contextVk->getRenderer();
 
     // Throttle the submissions to avoid getting too far ahead of the GPU.
-    SwapHistory &swap = mSwapHistory[mCurrentSwapHistoryIndex];
+    Serial *swapSerial = &mSwapHistory[mCurrentSwapHistoryIndex];
     {
         ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present: Throttle CPU");
-        if (swap.sharedFence.isReferenced())
-        {
-            // TODO: https://issuetracker.google.com/170312581 - This wait needs to be sure to
-            // happen after work has submitted
-            ANGLE_TRY(swap.waitFence(contextVk));
-            swap.destroy(renderer);
-        }
+        ANGLE_TRY(renderer->finishToSerial(contextVk, *swapSerial));
     }
 
     SwapchainImage &image               = mSwapchainImages[mCurrentSwapchainImageIndex];
@@ -1425,10 +1395,8 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
         presentInfo.pNext = &presentRegions;
     }
 
-    // Update the swap history for this presentation
-    // TODO: https://issuetracker.google.com/issues/170312581 - this will force us to flush worker
-    // queue to get the fence.
-    swap.sharedFence = contextVk->getLastSubmittedFence();
+    // TODO(jmadill): Fix potential serial race. b/172704839
+    *swapSerial = renderer->getLastSubmittedQueueSerial();
     ASSERT(!mAcquireImageSemaphore.valid());
 
     ++mCurrentSwapHistoryIndex;

src/libANGLE/renderer/vulkan/SurfaceVk.h

@@ -117,21 +117,6 @@ class OffscreenSurfaceVk : public SurfaceVk
 // Data structures used in WindowSurfaceVk
 namespace impl
 {
-// The submission fence of the context used to throttle the CPU.
-struct SwapHistory : angle::NonCopyable
-{
-    SwapHistory();
-    SwapHistory(SwapHistory &&other) = delete;
-    SwapHistory &operator=(SwapHistory &&other) = delete;
-    ~SwapHistory();
-
-    void destroy(RendererVk *renderer);
-
-    angle::Result waitFence(ContextVk *contextVk);
-
-    // Fence associated with the last submitted work to render to this swapchain image.
-    vk::Shared<vk::Fence> sharedFence;
-};
 static constexpr size_t kSwapHistorySize = 2;
 
 // Old swapchain and associated present semaphores that need to be scheduled for destruction when
@@ -317,9 +302,9 @@ class WindowSurfaceVk : public SurfaceVk
     VkSurfaceTransformFlagBitsKHR mEmulatedPreTransform;
     VkCompositeAlphaFlagBitsKHR mCompositeAlpha;
 
-    // A circular buffer that stores the submission fence of the context on every swap.  The CPU is
-    // throttled by waiting for the 2nd previous serial to finish.
-    std::array<impl::SwapHistory, impl::kSwapHistorySize> mSwapHistory;
+    // A circular buffer that stores the serial of the submission fence of the context on every
+    // swap. The CPU is throttled by waiting for the 2nd previous serial to finish.
+    std::array<Serial, impl::kSwapHistorySize> mSwapHistory;
     size_t mCurrentSwapHistoryIndex;
 
     // The previous swapchain which needs to be scheduled for destruction when appropriate.  This

src/libANGLE/renderer/vulkan/SyncVk.cpp

@@ -39,7 +39,6 @@ void SyncHelper::releaseToRenderer(RendererVk *renderer)
         ANGLE_TRACE_EVENT0("gpu.angle", "SyncHelper::releaseToRenderer");
         renderer->waitForCommandProcessorIdle(nullptr);
     }
-    mFence.reset(renderer->getDevice());
 }
 
 angle::Result SyncHelper::initialize(ContextVk *contextVk)
@@ -55,20 +54,6 @@ angle::Result SyncHelper::initialize(ContextVk *contextVk)
 
     DeviceScoped<Event> event(device);
     ANGLE_VK_TRY(contextVk, event.get().init(device, eventCreateInfo));
-    // TODO: https://issuetracker.google.com/170312581 - For now wait for worker thread to finish
-    // then get next fence from renderer
-    if (contextVk->getRenderer()->getFeatures().commandProcessor.enabled)
-    {
-        if (contextVk->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled)
-        {
-            contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
-        }
-        ANGLE_TRY(contextVk->getRenderer()->getNextSubmitFence(&mFence, false));
-    }
-    else
-    {
-        ANGLE_TRY(contextVk->getNextSubmitFence(&mFence));
-    }
 
     mEvent = event.release();
 
@@ -110,19 +95,18 @@ angle::Result SyncHelper::clientWait(Context *context,
         ANGLE_TRY(contextVk->flushImpl(nullptr));
     }
 
-    // TODO: https://issuetracker.google.com/170312581 - If we are using worker need to wait for the
-    // commands to be issued before waiting on the fence.
-    if (renderer->getFeatures().asynchronousCommandProcessing.enabled)
+    // Undefined behaviour. Early exit.
+    if (usedInRecordedCommands())
     {
-        ANGLE_TRACE_EVENT0("gpu.angle", "SyncHelper::clientWait");
-        renderer->waitForCommandProcessorIdle(contextVk);
+        WARN() << "Waiting on a sync that is not flushed";
+        *outResult = VK_TIMEOUT;
+        return angle::Result::Continue;
     }
 
-    // Wait on the fence that's expected to be signaled on the first vkQueueSubmit after
-    // `initialize` was called. The first fence is the fence created to signal this sync.
-    ASSERT(mFence.get().valid());
-    // TODO: https://issuetracker.google.com/170312581 - Wait could be command to worker
-    VkResult status = mFence.get().wait(renderer->getDevice(), timeout);
+    ASSERT(mUse.getSerial().valid());
+
+    VkResult status = VK_SUCCESS;
+    ANGLE_TRY(renderer->waitForSerialWithUserTimeout(context, mUse.getSerial(), timeout, &status));
 
     // Check for errors, but don't consider timeout as such.
     if (status != VK_TIMEOUT)
@@ -238,7 +222,7 @@ angle::Result SyncHelperNativeFence::initializeWithFd(ContextVk *contextVk, int
             submitInfo.sType        = VK_STRUCTURE_TYPE_SUBMIT_INFO;
 
             ANGLE_TRY(renderer->queueSubmit(contextVk, contextVk->getPriority(), submitInfo,
-                                            nullptr, &fence.get(), &serialOut));
+                                            vk::ResourceUseList(), &fence.get(), &serialOut));
         }
 
         VkFenceGetFdInfoKHR fenceGetFdInfo = {};

src/libANGLE/renderer/vulkan/SyncVk.h

@@ -55,9 +55,6 @@ class SyncHelper : public vk::Resource
     // The vkEvent that's signaled on `init` and can be waited on in `serverWait`, or queried with
     // `getStatus`.
     Event mEvent;
-    // The fence is signaled once the CB including the `init` signal is executed.
-    // `clientWait` waits on this fence.
-    Shared<Fence> mFence;
 };
 
 // Implementation of sync types: EGLSync(EGL_SYNC_ANDROID_NATIVE_FENCE_ANDROID).

src/libANGLE/renderer/vulkan/vk_helpers.h

@@ -1105,6 +1105,8 @@ class CommandBufferHelper : angle::NonCopyable
     const RenderPassDesc &getRenderPassDesc() const { return mRenderPassDesc; }
     const AttachmentOpsArray &getAttachmentOps() const { return mAttachmentOps; }
 
+    bool hasRenderPass() const { return mIsRenderPassCommandBuffer; }
+
   private:
     bool onDepthStencilAccess(ResourceAccess access,
                               uint32_t *cmdCountInvalidated,

src/tests/test_utils/angle_test_instantiate.cpp

@@ -515,9 +515,10 @@ bool IsConfigAllowlisted(const SystemInfo &systemInfo, const PlatformParameters
         switch (param.getRenderer())
         {
             case EGL_PLATFORM_ANGLE_TYPE_OPENGL_ANGLE:
-            case EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE:
-                // Note that system info collection depends on Vulkan support.
                 return true;
+            case EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE:
+                // http://issuetracker.google.com/173004081
+                return !IsIntel() || param.eglParameters.asyncCommandQueueFeatureVulkan != EGL_TRUE;
             default:
                 return false;
         }