Allow single threaded CommandProcessor

include/platform/FeaturesVk.h

@@ -336,14 +336,14 @@ struct FeaturesVk : FeatureSetBase
         "fine grained pipeline stage dependency information",
         &members, "http://anglebug.com/4633"};
 
-    // Enable parallel thread that processes and submits vulkan command buffers.
-    // Currently off by default to enable testing.
-    Feature enableCommandProcessingThread = {
-        "enable_command_processing_thread", FeatureCategory::VulkanFeatures,
-        "Enable parallel processing and submission of Vulkan commands in worker thread", &members,
-        "http://anglebug.com/4324"};
-
-    // Enable parallel thread execution when enableCommandProcessingThread is enabled.
+    // Tell Vulkan back-end to use CommandProcessor class to dispatch work to the GPU. The work will
+    // happen asynchronously in a different thread if asynchronousCommandProcessing is true.
+    // Otherwise use Renderer::CommandQueue to dispatch work.
+    Feature commandProcessor = {"command_processor", FeatureCategory::VulkanFeatures,
+                                "Use CommandProcessor class to dispatch work to GPU.", &members,
+                                "http://anglebug.com/4324"};
+
+    // Enable parallel thread execution when commandProcessor is enabled.
     // Currently off by default.
     Feature asynchronousCommandProcessing = {"asynchronous_command_processing",
                                              FeatureCategory::VulkanFeatures,

src/libANGLE/renderer/vulkan/CommandProcessor.h

@@ -27,10 +27,11 @@ class CommandProcessor;
 
 namespace vk
 {
-// CommandProcessorTask is used to queue a task to the worker thread when
-// enableCommandProcessingThread feature is true.
-// Issuing the CustomTask::Exit command will cause the worker thread to clean up it's resources and
-// shut down. This command is sent when the renderer instance shuts down. Custom tasks are:
+// CommandProcessor is used to dispatch work to the GPU when commandProcessor feature is true.
+// If asynchronousCommandProcessing is enabled the work will be queued and handled by a worker
+// thread asynchronous to the context. Issuing the CustomTask::Exit command will cause the worker
+// thread to clean up it's resources and shut down. This command is sent when the renderer instance
+// shuts down. Custom tasks are:
 
 enum CustomTask
 {
@@ -45,6 +46,10 @@ enum CustomTask
     FinishToSerial,
     // Execute QueuePresent
     Present,
+    // do cleanup processing on completed commands
+    // TODO: https://issuetracker.google.com/170312581 - should be able to remove
+    // checkCompletedCommands command with fence refactor.
+    CheckCompletedCommands,
     // Exit the command processor thread
     Exit,
 };
@@ -62,7 +67,7 @@ class CommandProcessorTask
                              CommandBufferHelper *commandBuffer,
                              vk::RenderPass *renderPass);
 
-    void initPresent(egl::ContextPriority priority, VkPresentInfoKHR presentInfo);
+    void initPresent(egl::ContextPriority priority, VkPresentInfoKHR &presentInfo);
 
     void initFinishToSerial(Serial serial);
 
@@ -104,6 +109,8 @@ class CommandProcessorTask
     ContextVk *getContextVk() const { return mContextVk; }
 
   private:
+    void copyPresentInfo(const VkPresentInfoKHR &other);
+
     CustomTask mTask;
 
     // ProcessCommands
@@ -123,6 +130,13 @@ class CommandProcessorTask
 
     // Present command data
     VkPresentInfoKHR mPresentInfo;
+    VkSwapchainKHR mSwapchain;
+    VkSemaphore mWaitSemaphore;
+    uint32_t mImageIndex;
+    // Used by Present if supportsIncrementalPresent is enabled
+    VkPresentRegionKHR mPresentRegion;
+    VkPresentRegionsKHR mPresentRegions;
+    std::vector<VkRectLayerKHR> mRects;
 
     // Used by OneOffQueueSubmit
     VkCommandBuffer mOneOffCommandBufferVk;
@@ -162,8 +176,6 @@ class TaskProcessor : angle::NonCopyable
     angle::Result releasePrimaryCommandBuffer(vk::Context *context,
                                               vk::PrimaryCommandBuffer &&commandBuffer);
 
-    void clearAllGarbage(vk::Context *context);
-
     angle::Result finishToSerial(vk::Context *context, Serial serial);
 
     VkResult present(VkQueue queue, const VkPresentInfoKHR &presentInfo);
@@ -183,25 +195,39 @@ class TaskProcessor : angle::NonCopyable
 
     vk::Shared<vk::Fence> getLastSubmittedFenceWithLock(VkDevice device) const;
 
-    // Check to see which batches have finished completion (forward progress for
-    // mLastCompletedQueueSerial, for example for when the application busy waits on a query
-    // result). It would be nice if we didn't have to expose this for QueryVk::getResult.
-    angle::Result checkCompletedCommands(vk::Context *context);
-
     void handleDeviceLost(vk::Context *context);
 
+    // Called by CommandProcessor to process any completed work
+    angle::Result lockAndCheckCompletedCommands(vk::Context *context);
+
+    VkResult getLastAndClearPresentResult(VkSwapchainKHR swapchain);
+
   private:
+    bool isValidWorkerThread(vk::Context *context) const;
+
     angle::Result releaseToCommandBatch(vk::Context *context,
                                         vk::PrimaryCommandBuffer &&commandBuffer,
                                         vk::CommandPool *commandPool,
                                         vk::CommandBatch *batch);
 
+    // Check to see which batches have finished completion (forward progress for
+    // mLastCompletedQueueSerial, for example for when the application busy waits on a query
+    // result). It would be nice if we didn't have to expose this for QueryVk::getResult.
+    angle::Result checkCompletedCommandsNoLock(vk::Context *context);
+
     vk::GarbageQueue mGarbageQueue;
+
+    mutable std::mutex mInFlightCommandsMutex;
     std::vector<vk::CommandBatch> mInFlightCommands;
 
     // Keeps a free list of reusable primary command buffers.
     vk::PersistentCommandPool mPrimaryCommandPool;
     std::thread::id mThreadId;
+
+    // Track present info
+    std::mutex mSwapchainStatusMutex;
+    std::condition_variable mSwapchainStatusCondition;
+    std::map<VkSwapchainKHR, VkResult> mSwapchainStatus;
 };
 
 class CommandProcessor : public vk::Context
@@ -210,6 +236,8 @@ class CommandProcessor : public vk::Context
     CommandProcessor(RendererVk *renderer);
     ~CommandProcessor() override;
 
+    angle::Result initTaskProcessor(vk::Context *context);
+
     void handleError(VkResult result,
                      const char *file,
                      const char *function,
@@ -223,6 +251,8 @@ class CommandProcessor : public vk::Context
     // thread
     void queueCommand(vk::Context *context, vk::CommandProcessorTask *task);
 
+    void checkCompletedCommands(vk::Context *context);
+
     // Used by main thread to wait for worker thread to complete all outstanding work.
     void waitForWorkComplete(vk::Context *context);
     Serial getCurrentQueueSerial();
@@ -231,7 +261,7 @@ class CommandProcessor : public vk::Context
     // Wait until desired serial has been processed.
     void finishToSerial(vk::Context *context, Serial serial);
 
-    vk::Shared<vk::Fence> getLastSubmittedFence() const;
+    vk::Shared<vk::Fence> getLastSubmittedFence(const vk::Context *context) const;
     void handleDeviceLost();
 
     bool hasPendingError() const
@@ -246,13 +276,18 @@ class CommandProcessor : public vk::Context
 
     void finishAllWork(vk::Context *context);
 
+    VkResult getLastPresentResult(VkSwapchainKHR swapchain)
+    {
+        return mTaskProcessor.getLastAndClearPresentResult(swapchain);
+    }
+
   private:
     // Command processor thread, called by processTasks. The loop waits for work to
     // be submitted from a separate thread.
     angle::Result processTasksImpl(bool *exitThread);
 
     // Command processor thread, process a task
-    angle::Result processTask(vk::CommandProcessorTask *task);
+    angle::Result processTask(vk::Context *context, vk::CommandProcessorTask *task);
 
     std::queue<vk::CommandProcessorTask> mTasks;
     mutable std::mutex mWorkerMutex;

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -399,7 +399,8 @@ angle::Result CommandQueue::init(vk::Context *context)
 
 angle::Result CommandQueue::checkCompletedCommands(vk::Context *context)
 {
-    ANGLE_TRACE_EVENT0("gpu.angle", "CommandQueue::checkCompletedCommands");
+    ANGLE_TRACE_EVENT0("gpu.angle", "CommandQueue::checkCompletedCommandsNoLock");
+    ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
     RendererVk *renderer = context->getRenderer();
     VkDevice device      = renderer->getDevice();
 
@@ -522,6 +523,7 @@ angle::Result CommandQueue::allocatePrimaryCommandBuffer(vk::Context *context,
 angle::Result CommandQueue::releasePrimaryCommandBuffer(vk::Context *context,
                                                         vk::PrimaryCommandBuffer &&commandBuffer)
 {
+    ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
     ASSERT(mPrimaryCommandPool.valid());
     ANGLE_TRY(mPrimaryCommandPool.collect(context, std::move(commandBuffer)));
 
@@ -558,7 +560,7 @@ angle::Result CommandQueue::finishToSerial(vk::Context *context,
                                            Serial finishSerial,
                                            uint64_t timeout)
 {
-    ASSERT(!context->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled);
+    ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
 
     if (mInFlightCommands.empty())
     {
@@ -612,6 +614,7 @@ angle::Result CommandQueue::submitFrame(vk::Context *context,
                                         vk::PrimaryCommandBuffer &&commandBuffer)
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "CommandQueue::submitFrame");
+    ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
 
     RendererVk *renderer = context->getRenderer();
     VkDevice device      = renderer->getDevice();
@@ -649,7 +652,7 @@ angle::Result CommandQueue::submitFrame(vk::Context *context,
 
 vk::Shared<vk::Fence> CommandQueue::getLastSubmittedFence(const vk::Context *context) const
 {
-    ASSERT(!context->getRenderer()->getFeatures().enableCommandProcessingThread.enabled);
+    ASSERT(!context->getRenderer()->getFeatures().commandProcessor.enabled);
 
     vk::Shared<vk::Fence> fence;
     if (!mInFlightCommands.empty())
@@ -844,6 +847,9 @@ void ContextVk::onDestroy(const gl::Context *context)
     mGpuEventQueryPool.destroy(device);
     mCommandPool.destroy(device);
     mPrimaryCommands.destroy(device);
+
+    // This will clean up any outstanding buffer allocations
+    (void)mRenderer->cleanupGarbage(false);
 }
 
 angle::Result ContextVk::getIncompleteTexture(const gl::Context *context,
@@ -1861,7 +1867,7 @@ angle::Result ContextVk::submitFrame(const VkSubmitInfo &submitInfo,
                                      vk::ResourceUseList *resourceList,
                                      vk::PrimaryCommandBuffer &&commandBuffer)
 {
-    ASSERT(!getRenderer()->getFeatures().enableCommandProcessingThread.enabled);
+    ASSERT(!getRenderer()->getFeatures().commandProcessor.enabled);
 
     if (vk::CommandBufferHelper::kEnableCommandStreamDiagnostics)
     {
@@ -2203,7 +2209,7 @@ void ContextVk::clearAllGarbage()
         garbage.destroy(mRenderer);
     }
     mCurrentGarbage.clear();
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         // Issue command to CommandProcessor to ensure all work is complete, which will return any
         // garbage items as well.
@@ -2220,7 +2226,7 @@ void ContextVk::handleDeviceLost()
     mOutsideRenderPassCommands->reset();
     mRenderPassCommands->reset();
 
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         mRenderer->handleDeviceLost();
     }
@@ -3410,9 +3416,10 @@ angle::Result ContextVk::onMakeCurrent(const gl::Context *context)
 angle::Result ContextVk::onUnMakeCurrent(const gl::Context *context)
 {
     ANGLE_TRY(flushImpl(nullptr));
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
-        mRenderer->waitForCommandProcessorIdle(this);
+        ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::onUnMakeCurrent");
+        mRenderer->finishAllWork(this);
     }
     mCurrentWindowSurface = nullptr;
     return angle::Result::Continue;
@@ -4427,12 +4434,9 @@ angle::Result ContextVk::flushImpl(const vk::Semaphore *signalSemaphore)
     mDefaultUniformStorage.releaseInFlightBuffersToResourceUseList(this);
     mStagingBuffer.releaseInFlightBuffersToResourceUseList(this);
 
-    // TODO: https://issuetracker.google.com/issues/170329600 - Verify that
-    // waitForSwapchainImageIfNecessary makes sense both w/ & w/o threading. I believe they do, but
-    // want confirmation.
     waitForSwapchainImageIfNecessary();
 
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         // Some tasks from ContextVk::submitFrame() that run before CommandQueue::submitFrame()
         gl::RunningGraphWidget *renderPassCount =
@@ -4506,7 +4510,7 @@ angle::Result ContextVk::finishImpl()
 
     ANGLE_TRY(flushImpl(nullptr));
 
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         ANGLE_TRY(finishToSerial(getLastSubmittedQueueSerial()));
     }
@@ -4555,14 +4559,21 @@ bool ContextVk::isSerialInUse(Serial serial) const
 
 angle::Result ContextVk::checkCompletedCommands()
 {
-    ASSERT(!mRenderer->getFeatures().enableCommandProcessingThread.enabled);
+    if (mRenderer->getFeatures().commandProcessor.enabled)
+    {
+        // TODO: https://issuetracker.google.com/169788986 - would be better if we could just wait
+        // for the work we need but that requires QueryHelper to use the actual serial for the
+        // query.
+        mRenderer->checkCompletedCommands(this);
+        return angle::Result::Continue;
+    }
 
     return mCommandQueue.checkCompletedCommands(this);
 }
 
 angle::Result ContextVk::finishToSerial(Serial serial)
 {
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         mRenderer->finishToSerial(this, serial);
         return angle::Result::Continue;
@@ -4596,7 +4607,7 @@ angle::Result ContextVk::ensureSubmitFenceInitialized()
 
 angle::Result ContextVk::getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOut)
 {
-    ASSERT(!getRenderer()->getFeatures().enableCommandProcessingThread.enabled);
+    ASSERT(!getRenderer()->getFeatures().commandProcessor.enabled);
 
     ANGLE_TRY(ensureSubmitFenceInitialized());
 
@@ -4607,9 +4618,9 @@ angle::Result ContextVk::getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOu
 
 vk::Shared<vk::Fence> ContextVk::getLastSubmittedFence() const
 {
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
-        return mRenderer->getLastSubmittedFence();
+        return mRenderer->getLastSubmittedFence(this);
     }
     return mCommandQueue.getLastSubmittedFence(this);
 }
@@ -5020,7 +5031,7 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
 
     vk::RenderPass *renderPass = nullptr;
     ANGLE_TRY(mRenderPassCommands->getRenderPassWithOps(this, &renderPass));
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         mRenderPassCommands->markClosed();
         vk::CommandProcessorTask flushToPrimary;
@@ -5170,7 +5181,7 @@ angle::Result ContextVk::flushOutsideRenderPassCommands()
 
     vk::RenderPass *renderPass = nullptr;
     ANGLE_TRY(mOutsideRenderPassCommands->getRenderPassWithOps(this, &renderPass));
-    if (mRenderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (mRenderer->getFeatures().commandProcessor.enabled)
     {
         mOutsideRenderPassCommands->markClosed();
         vk::CommandProcessorTask flushToPrimary;

src/libANGLE/renderer/vulkan/QueryVk.cpp

@@ -168,6 +168,8 @@ angle::Result QueryVk::queryCounter(const gl::Context *context)
 
 angle::Result QueryVk::getResult(const gl::Context *context, bool wait)
 {
+    ANGLE_TRACE_EVENT0("gpu.angle", "QueryVk::getResult");
+
     if (mCachedResultValid)
     {
         return angle::Result::Continue;
@@ -182,11 +184,10 @@ angle::Result QueryVk::getResult(const gl::Context *context, bool wait)
     // has pending work should flush begin too.
     // TODO: https://issuetracker.google.com/169788986 - can't guarantee hasPendingWork() works when
     // using threaded worker
-    if (mQueryHelper.hasPendingWork(contextVk) ||
-        contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+    if (mQueryHelper.hasPendingWork(contextVk))
     {
         ANGLE_TRY(contextVk->flushImpl(nullptr));
-        if (contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+        if (contextVk->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled)
         {
             // TODO: https://issuetracker.google.com/170312581 - For now just stalling here
             contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
@@ -196,14 +197,12 @@ angle::Result QueryVk::getResult(const gl::Context *context, bool wait)
         ASSERT(!mQueryHelper.hasPendingWork(contextVk));
     }
 
-    if (!contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
-    {
-        // If the command buffer this query is being written to is still in flight, its reset
-        // command may not have been performed by the GPU yet.  To avoid a race condition in this
-        // case, wait for the batch to finish first before querying (or return not-ready if not
-        // waiting).
-        ANGLE_TRY(contextVk->checkCompletedCommands());
-    }
+    ANGLE_TRY(contextVk->checkCompletedCommands());
+
+    // If the command buffer this query is being written to is still in flight, its reset
+    // command may not have been performed by the GPU yet.  To avoid a race condition in this
+    // case, wait for the batch to finish first before querying (or return not-ready if not
+    // waiting).
     if (contextVk->isSerialInUse(mQueryHelper.getStoredQueueSerial()))
     {
         if (!wait)

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -505,7 +505,7 @@ void RendererVk::releaseSharedResources(vk::ResourceUseList *resourceList)
 
 void RendererVk::onDestroy()
 {
-    if (getFeatures().enableCommandProcessingThread.enabled)
+    if (getFeatures().commandProcessor.enabled)
     {
         // Shutdown worker thread
         mCommandProcessor.shutdown(&mCommandProcessorThread);
@@ -914,11 +914,19 @@ angle::Result RendererVk::initialize(DisplayVk *displayVk,
 
     setGlobalDebugAnnotator();
 
-    if (getFeatures().enableCommandProcessingThread.enabled)
+    if (getFeatures().commandProcessor.enabled)
     {
-        mCommandProcessorThread =
-            std::thread(&vk::CommandProcessor::processTasks, &mCommandProcessor);
-        mCommandProcessor.waitForWorkComplete(nullptr);
+        if (getFeatures().asynchronousCommandProcessing.enabled)
+        {
+            ASSERT(getFeatures().commandProcessor.enabled);
+            mCommandProcessorThread =
+                std::thread(&vk::CommandProcessor::processTasks, &mCommandProcessor);
+            waitForCommandProcessorIdle(displayVk);
+        }
+        else
+        {
+            ANGLE_TRY(mCommandProcessor.initTaskProcessor(displayVk));
+        }
     }
 
     return angle::Result::Continue;
@@ -1931,7 +1939,7 @@ void RendererVk::initFeatures(DisplayVk *displayVk, const ExtensionNameList &dev
     ANGLE_FEATURE_CONDITION(&mFeatures, preferAggregateBarrierCalls, isNvidia || isAMD || isIntel);
 
     // Currently disabled by default: http://anglebug.com/4324
-    ANGLE_FEATURE_CONDITION(&mFeatures, enableCommandProcessingThread, false);
+    ANGLE_FEATURE_CONDITION(&mFeatures, commandProcessor, false);
 
     // Currently disabled by default: http://anglebug.com/4324
     ANGLE_FEATURE_CONDITION(&mFeatures, asynchronousCommandProcessing, false);
@@ -2238,7 +2246,7 @@ angle::Result RendererVk::queueSubmit(vk::Context *context,
         outputVmaStatString();
     }
 
-    ASSERT(!getFeatures().enableCommandProcessingThread.enabled);
+    ASSERT(!getFeatures().commandProcessor.enabled);
 
     {
         std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
@@ -2266,15 +2274,20 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
                                             const vk::Fence *fence,
                                             Serial *serialOut)
 {
-    if (getFeatures().enableCommandProcessingThread.enabled)
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::queueSubmitOneOff");
+
+    if (getFeatures().commandProcessor.enabled)
     {
         vk::CommandProcessorTask oneOffQueueSubmit;
         oneOffQueueSubmit.initOneOffQueueSubmit(primary.getHandle(), priority, fence);
         queueCommand(context, &oneOffQueueSubmit);
-        waitForCommandProcessorIdle(context);
+        // TODO: https://issuetracker.google.com/170312581 - should go away with improved fence
+        // management
+        if (getFeatures().asynchronousCommandProcessing.enabled)
+        {
+            waitForCommandProcessorIdle(context);
+        }
         *serialOut = getLastSubmittedQueueSerial();
-
-        ANGLE_TRY(cleanupGarbage(false));
     }
     else
     {
@@ -2294,7 +2307,8 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
 
 angle::Result RendererVk::queueWaitIdle(vk::Context *context, egl::ContextPriority priority)
 {
-    if (getFeatures().enableCommandProcessingThread.enabled)
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::queueWaitIdle");
+    if (getFeatures().asynchronousCommandProcessing.enabled)
     {
         // Wait for all pending commands to get sent before issuing vkQueueWaitIdle
         waitForCommandProcessorIdle(context);
@@ -2311,7 +2325,9 @@ angle::Result RendererVk::queueWaitIdle(vk::Context *context, egl::ContextPriori
 
 angle::Result RendererVk::deviceWaitIdle(vk::Context *context)
 {
-    if (getFeatures().enableCommandProcessingThread.enabled)
+    ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::deviceWaitIdle");
+
+    if (getFeatures().asynchronousCommandProcessing.enabled)
     {
         // Wait for all pending commands to get sent before issuing vkQueueWaitIdle
         waitForCommandProcessorIdle(context);
@@ -2331,7 +2347,7 @@ VkResult RendererVk::queuePresent(egl::ContextPriority priority,
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::queuePresent");
 
-    ASSERT(!getFeatures().enableCommandProcessingThread.enabled);
+    ASSERT(!getFeatures().commandProcessor.enabled);
 
     std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
 

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -227,9 +227,9 @@ class RendererVk : angle::NonCopyable
         }
     }
 
-    vk::Shared<vk::Fence> getLastSubmittedFence() const
+    vk::Shared<vk::Fence> getLastSubmittedFence(const vk::Context *context) const
     {
-        return mCommandProcessor.getLastSubmittedFence();
+        return mCommandProcessor.getLastSubmittedFence(context);
     }
     void handleDeviceLost() { mCommandProcessor.handleDeviceLost(); }
 
@@ -250,7 +250,7 @@ class RendererVk : angle::NonCopyable
 
     ANGLE_INLINE Serial getCurrentQueueSerial()
     {
-        if (getFeatures().enableCommandProcessingThread.enabled)
+        if (getFeatures().commandProcessor.enabled)
         {
             return mCommandProcessor.getCurrentQueueSerial();
         }
@@ -259,7 +259,7 @@ class RendererVk : angle::NonCopyable
     }
     ANGLE_INLINE Serial getLastSubmittedQueueSerial()
     {
-        if (getFeatures().enableCommandProcessingThread.enabled)
+        if (getFeatures().commandProcessor.enabled)
         {
             return mCommandProcessor.getLastSubmittedSerial();
         }
@@ -274,6 +274,11 @@ class RendererVk : angle::NonCopyable
 
     void onCompletedSerial(Serial serial);
 
+    VkResult getLastPresentResult(VkSwapchainKHR swapchain)
+    {
+        return mCommandProcessor.getLastPresentResult(swapchain);
+    }
+
     bool enableDebugUtils() const { return mEnableDebugUtils; }
 
     SamplerCache &getSamplerCache() { return mSamplerCache; }
@@ -289,6 +294,7 @@ class RendererVk : angle::NonCopyable
     vk::Error getAndClearPendingError() { return mCommandProcessor.getAndClearPendingError(); }
     void waitForCommandProcessorIdle(vk::Context *context)
     {
+        ASSERT(getFeatures().asynchronousCommandProcessing.enabled);
         mCommandProcessor.waitForWorkComplete(context);
     }
 
@@ -297,6 +303,11 @@ class RendererVk : angle::NonCopyable
         mCommandProcessor.finishToSerial(context, serial);
     }
 
+    void checkCompletedCommands(vk::Context *context)
+    {
+        mCommandProcessor.checkCompletedCommands(context);
+    }
+
     void finishAllWork(vk::Context *context) { mCommandProcessor.finishAllWork(context); }
     VkQueue getVkQueue(egl::ContextPriority priority) const { return mQueues[priority]; }
 
@@ -308,6 +319,8 @@ class RendererVk : angle::NonCopyable
 
     void outputVmaStatString();
 
+    angle::Result cleanupGarbage(bool block);
+
   private:
     angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
@@ -327,8 +340,6 @@ class RendererVk : angle::NonCopyable
     template <VkFormatFeatureFlags VkFormatProperties::*features>
     bool hasFormatFeatureBits(VkFormat format, const VkFormatFeatureFlags featureBits) const;
 
-    angle::Result cleanupGarbage(bool block);
-
     egl::Display *mDisplay;
 
     mutable bool mCapsInitialized;

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -464,6 +464,8 @@ void SwapHistory::destroy(RendererVk *renderer)
 angle::Result SwapHistory::waitFence(ContextVk *contextVk)
 {
     ASSERT(sharedFence.isReferenced());
+    // TODO: https://issuetracker.google.com/170312581 - This wait needs to be synchronized with
+    // worker thread
     ANGLE_VK_TRY(contextVk, sharedFence.get().wait(contextVk->getDevice(),
                                                    std::numeric_limits<uint64_t>::max()));
     return angle::Result::Continue;
@@ -1224,6 +1226,35 @@ egl::Error WindowSurfaceVk::swap(const gl::Context *context)
     return angle::ToEGL(result, displayVk, EGL_BAD_SURFACE);
 }
 
+angle::Result WindowSurfaceVk::computePresentOutOfDate(vk::Context *context,
+                                                       VkResult result,
+                                                       bool *presentOutOfDate)
+{
+    // If OUT_OF_DATE is returned, it's ok, we just need to recreate the swapchain before
+    // continuing.
+    // If VK_SUBOPTIMAL_KHR is returned it's because the device orientation changed and we should
+    // recreate the swapchain with a new window orientation.
+    if (context->getRenderer()->getFeatures().enablePreRotateSurfaces.enabled)
+    {
+        // Also check for VK_SUBOPTIMAL_KHR.
+        *presentOutOfDate = ((result == VK_ERROR_OUT_OF_DATE_KHR) || (result == VK_SUBOPTIMAL_KHR));
+        if (!*presentOutOfDate)
+        {
+            ANGLE_VK_TRY(context, result);
+        }
+    }
+    else
+    {
+        // We aren't quite ready for that so just ignore for now.
+        *presentOutOfDate = result == VK_ERROR_OUT_OF_DATE_KHR;
+        if (!*presentOutOfDate && result != VK_SUBOPTIMAL_KHR)
+        {
+            ANGLE_VK_TRY(context, result);
+        }
+    }
+    return angle::Result::Continue;
+}
+
 angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
                                        EGLint *rects,
                                        EGLint n_rects,
@@ -1239,6 +1270,8 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
         ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present: Throttle CPU");
         if (swap.sharedFence.isReferenced())
         {
+            // TODO: https://issuetracker.google.com/170312581 - This wait needs to be sure to
+            // happen after work has submitted
             ANGLE_TRY(swap.waitFence(contextVk));
             swap.destroy(renderer);
         }
@@ -1354,7 +1387,7 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
         presentRegion.pRectangles = vkRects.data();
 
         presentRegions.sType          = VK_STRUCTURE_TYPE_PRESENT_REGIONS_KHR;
-        presentRegions.pNext          = nullptr;
+        presentRegions.pNext          = presentInfo.pNext;
         presentRegions.swapchainCount = 1;
         presentRegions.pRegions       = &presentRegion;
 
@@ -1372,63 +1405,24 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
         mCurrentSwapHistoryIndex == mSwapHistory.size() ? 0 : mCurrentSwapHistoryIndex;
 
     VkResult result;
-    if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (renderer->getFeatures().commandProcessor.enabled)
     {
         vk::CommandProcessorTask present;
         present.initPresent(contextVk->getPriority(), presentInfo);
 
-        // Make sure everything has been submitted (and errors handled)
-        renderer->waitForCommandProcessorIdle(contextVk);
-
-        // Submit queuePresent all by itself (ignoring interference from other threads for now)
+        ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present");
         renderer->queueCommand(contextVk, &present);
-        // TODO: https://issuetracker.google.com/issues/170329600 - Just stalling here for now, but
-        // really want to let main thread continue
-        // need to figure out how to handle work below off-thread and sync to main
-        // Also, need to fix lifetime of presentInfo data when main thread continues.
-        // There is a bunch of work happening after present to deal with swapchain recreation.
-        // Will that require moving a large chunk of swapImpl to the CommandProcessor?
-        // That will likely require serializing access to the WindowSurfaceVk object in order
-        // to have current content.
+        // Always return success, when we call acquireNextImage we'll check the return code. This
+        // allows the app to continue working until we really need to know the return code from
+        // present.
         result = VK_SUCCESS;
-
-        // wait for the queuePresent to be submitted and intentionally set the context to nullptr so
-        // that we can catch any error. Note this doesn't prevent another context from grabbing the
-        // error. Will be fixed properly in a follow-up as part of present work.
-        renderer->waitForCommandProcessorIdle(nullptr);
-        if (renderer->hasPendingError())
-        {
-            vk::Error error = renderer->getAndClearPendingError();
-            result          = error.mErrorCode;
-        }
     }
     else
     {
         result = renderer->queuePresent(contextVk->getPriority(), presentInfo);
     }
 
-    // If OUT_OF_DATE is returned, it's ok, we just need to recreate the swapchain before
-    // continuing.
-    // If VK_SUBOPTIMAL_KHR is returned it's because the device orientation changed and we should
-    // recreate the swapchain with a new window orientation.
-    if (renderer->getFeatures().enablePreRotateSurfaces.enabled)
-    {
-        // Also check for VK_SUBOPTIMAL_KHR.
-        *presentOutOfDate = ((result == VK_ERROR_OUT_OF_DATE_KHR) || (result == VK_SUBOPTIMAL_KHR));
-        if (!*presentOutOfDate)
-        {
-            ANGLE_VK_TRY(contextVk, result);
-        }
-    }
-    else
-    {
-        // We aren't quite ready for that so just ignore for now.
-        *presentOutOfDate = result == VK_ERROR_OUT_OF_DATE_KHR;
-        if (!*presentOutOfDate && result != VK_SUBOPTIMAL_KHR)
-        {
-            ANGLE_VK_TRY(contextVk, result);
-        }
-    }
+    ANGLE_TRY(computePresentOutOfDate(contextVk, result, presentOutOfDate));
 
     return angle::Result::Continue;
 }
@@ -1488,6 +1482,15 @@ angle::Result WindowSurfaceVk::doDeferredAcquireNextImage(const gl::Context *con
     ContextVk *contextVk = vk::GetImpl(context);
     DisplayVk *displayVk = vk::GetImpl(context->getDisplay());
 
+    if (contextVk->getFeatures().commandProcessor.enabled)
+    {
+        VkResult result = contextVk->getRenderer()->getLastPresentResult(mSwapchain);
+
+        // Now that we have the result from the last present need to determine if it's out of date
+        // or not.
+        ANGLE_TRY(computePresentOutOfDate(contextVk, result, &presentOutOfDate));
+    }
+
     ANGLE_TRY(checkForOutOfDateSwapchain(contextVk, presentOutOfDate));
 
     {

src/libANGLE/renderer/vulkan/SurfaceVk.h

@@ -289,6 +289,9 @@ class WindowSurfaceVk : public SurfaceVk
     // Called when a swapchain image whose acquisition was deferred must be acquired.  This method
     // will recreate the swapchain (if needed) and call the acquireNextSwapchainImage() method.
     angle::Result doDeferredAcquireNextImage(const gl::Context *context, bool presentOutOfDate);
+    angle::Result computePresentOutOfDate(vk::Context *context,
+                                          VkResult result,
+                                          bool *presentOutOfDate);
     angle::Result present(ContextVk *contextVk,
                           EGLint *rects,
                           EGLint n_rects,

src/libANGLE/renderer/vulkan/SyncVk.cpp

@@ -34,8 +34,9 @@ void SyncHelper::releaseToRenderer(RendererVk *renderer)
     renderer->collectGarbageAndReinit(&mUse, &mEvent);
     // TODO: https://issuetracker.google.com/170312581 - Currently just stalling on worker thread
     // here to try and avoid race condition. If this works, need some alternate solution
-    if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+    if (renderer->getFeatures().asynchronousCommandProcessing.enabled)
     {
+        ANGLE_TRACE_EVENT0("gpu.angle", "SyncHelper::releaseToRenderer");
         renderer->waitForCommandProcessorIdle(nullptr);
     }
     mFence.reset(renderer->getDevice());
@@ -56,9 +57,12 @@ angle::Result SyncHelper::initialize(ContextVk *contextVk)
     ANGLE_VK_TRY(contextVk, event.get().init(device, eventCreateInfo));
     // TODO: https://issuetracker.google.com/170312581 - For now wait for worker thread to finish
     // then get next fence from renderer
-    if (contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+    if (contextVk->getRenderer()->getFeatures().commandProcessor.enabled)
     {
-        contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
+        if (contextVk->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled)
+        {
+            contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
+        }
         ANGLE_TRY(contextVk->getRenderer()->getNextSubmitFence(&mFence, false));
     }
     else
@@ -106,10 +110,11 @@ angle::Result SyncHelper::clientWait(Context *context,
         ANGLE_TRY(contextVk->flushImpl(nullptr));
     }
 
-    // If we are using worker need to wait for the commands to be issued before waiting on the
-    // fence.
-    if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+    // TODO: https://issuetracker.google.com/170312581 - If we are using worker need to wait for the
+    // commands to be issued before waiting on the fence.
+    if (renderer->getFeatures().asynchronousCommandProcessing.enabled)
     {
+        ANGLE_TRACE_EVENT0("gpu.angle", "SyncHelper::clientWait");
         renderer->waitForCommandProcessorIdle(contextVk);
     }
 
@@ -213,14 +218,18 @@ angle::Result SyncHelperNativeFence::initializeWithFd(ContextVk *contextVk, int
 
         retain(&contextVk->getResourceUseList());
 
-        if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+        if (renderer->getFeatures().commandProcessor.enabled)
         {
             CommandProcessorTask oneOffQueueSubmit;
             oneOffQueueSubmit.initOneOffQueueSubmit(VK_NULL_HANDLE, contextVk->getPriority(),
                                                     &fence.get());
             renderer->queueCommand(contextVk, &oneOffQueueSubmit);
             // TODO: https://issuetracker.google.com/170312581 - wait for now
-            renderer->waitForCommandProcessorIdle(contextVk);
+            if (renderer->getFeatures().asynchronousCommandProcessing.enabled)
+            {
+                ANGLE_TRACE_EVENT0("gpu.angle", "SyncHelperNativeFence::initializeWithFd");
+                renderer->waitForCommandProcessorIdle(contextVk);
+            }
         }
         else
         {
@@ -291,10 +300,11 @@ angle::Result SyncHelperNativeFence::clientWait(Context *context,
         ANGLE_TRY(contextVk->flushImpl(nullptr));
     }
 
-    // If we are using worker need to wait for the commands to be issued before waiting on the
-    // fence.
+    // TODO: https://issuetracker.google.com/170312581 - If we are using worker need to wait for the
+    // commands to be issued before waiting on the fence.
     if (contextVk->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled)
     {
+        ANGLE_TRACE_EVENT0("gpu.angle", "SyncHelperNativeFence::clientWait");
         contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
     }
 

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -2324,8 +2324,9 @@ void QueryHelper::writeTimestamp(ContextVk *contextVk, CommandBuffer *commandBuf
 
 bool QueryHelper::hasPendingWork(ContextVk *contextVk)
 {
-    // If the renderer has a queue serial higher than the stored one, the command buffers that
-    // recorded this query have already been submitted, so there is no pending work.
+    // TODO: https://issuetracker.google.com/169788986 - this is not a valid statement with
+    // CommandProcessor: If the renderer has a queue serial higher than the stored one, the command
+    // buffers that recorded this query have already been submitted, so there is no pending work.
     return mMostRecentSerial.valid() && (mMostRecentSerial == contextVk->getCurrentQueueSerial());
 }