Vulkan: functionally complete worker thread

include/platform/FeaturesVk.h

@@ -343,6 +343,13 @@ struct FeaturesVk : FeatureSetBase
         "Enable parallel processing and submission of Vulkan commands in worker thread", &members,
         "http://anglebug.com/4324"};
 
+    // Enable parallel thread execution when enableCommandProcessingThread is enabled.
+    // Currently off by default.
+    Feature asynchronousCommandProcessing = {"asynchronous_command_processing",
+                                             FeatureCategory::VulkanFeatures,
+                                             "Enable/Disable parallel processing of worker thread",
+                                             &members, "http://anglebug.com/4324"};
+
     // Whether the VkDevice supports the VK_KHR_shader_float16_int8 extension and has the
     // shaderFloat16 feature.
     Feature supportsShaderFloat16 = {"supports_shader_float16", FeatureCategory::VulkanFeatures,

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -47,7 +47,6 @@ class CommandQueue final : angle::NonCopyable
     bool hasInFlightCommands() const;
 
     angle::Result allocatePrimaryCommandBuffer(vk::Context *context,
-                                               const vk::CommandPool &commandPool,
                                                vk::PrimaryCommandBuffer *commandBufferOut);
     angle::Result releasePrimaryCommandBuffer(vk::Context *context,
                                               vk::PrimaryCommandBuffer &&commandBuffer);
@@ -653,11 +652,10 @@ class ContextVk : public ContextImpl, public vk::Context
     void updateOverlayOnPresent();
     void addOverlayUsedBuffersCount(vk::CommandBufferHelper *commandBuffer);
 
-    // Submit commands to worker thread for processing
-    ANGLE_INLINE void queueCommandsToWorker(const vk::CommandProcessorTask &commands)
-    {
-        mRenderer->queueCommands(commands);
-    }
+    // Sync any errors from the command processor
+    void commandProcessorSyncErrors();
+    // Sync any error from worker thread and queue up next command for processing
+    void commandProcessorSyncErrorsAndQueueCommand(vk::CommandProcessorTask *command);
     // When worker thread completes, it releases command buffers back to context queue
     void recycleCommandBuffer(vk::CommandBufferHelper *commandBuffer);
 
@@ -1085,6 +1083,7 @@ class ContextVk : public ContextImpl, public vk::Context
     // We use a single pool for recording commands. We also keep a free list for pool recycling.
     vk::CommandPool mCommandPool;
 
+    // TODO: This can be killed once threading is enabled https://issuetracker.google.com/153666475
     CommandQueue mCommandQueue;
     vk::GarbageList mCurrentGarbage;
 

src/libANGLE/renderer/vulkan/QueryVk.cpp

@@ -180,18 +180,30 @@ angle::Result QueryVk::getResult(const gl::Context *context, bool wait)
     // finite time.
     // Note regarding time-elapsed: end should have been called after begin, so flushing when end
     // has pending work should flush begin too.
-    if (mQueryHelper.hasPendingWork(contextVk))
+    // TODO: https://issuetracker.google.com/169788986 - can't guarantee hasPendingWork() works when
+    // using threaded worker
+    if (mQueryHelper.hasPendingWork(contextVk) ||
+        contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
     {
         ANGLE_TRY(contextVk->flushImpl(nullptr));
+        if (contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+        {
+            // TODO: https://issuetracker.google.com/170312581 - For now just stalling here
+            contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
+        }
 
         ASSERT(!mQueryHelperTimeElapsedBegin.hasPendingWork(contextVk));
         ASSERT(!mQueryHelper.hasPendingWork(contextVk));
     }
 
-    // If the command buffer this query is being written to is still in flight, its reset command
-    // may not have been performed by the GPU yet.  To avoid a race condition in this case, wait
-    // for the batch to finish first before querying (or return not-ready if not waiting).
-    ANGLE_TRY(contextVk->checkCompletedCommands());
+    if (!contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+    {
+        // If the command buffer this query is being written to is still in flight, its reset
+        // command may not have been performed by the GPU yet.  To avoid a race condition in this
+        // case, wait for the batch to finish first before querying (or return not-ready if not
+        // waiting).
+        ANGLE_TRY(contextVk->checkCompletedCommands());
+    }
     if (contextVk->isSerialInUse(mQueryHelper.getStoredQueueSerial()))
     {
         if (!wait)

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -467,6 +467,7 @@ RendererVk::RendererVk()
       mPipelineCacheVkUpdateTimeout(kPipelineCacheVkUpdatePeriod),
       mPipelineCacheDirty(false),
       mPipelineCacheInitialized(false),
+      mCommandProcessor(this),
       mGlslangInitialized(false)
 {
     VkFormatProperties invalid = {0, 0, kInvalidFormatFeatureFlags};
@@ -536,6 +537,10 @@ void RendererVk::onDestroy()
         std::lock_guard<std::mutex> lock(mFenceRecyclerMutex);
         mFenceRecycler.destroy(mDevice);
     }
+    {
+        std::lock_guard<decltype(mNextSubmitFenceMutex)> lock(mNextSubmitFenceMutex);
+        mNextSubmitFence.reset(mDevice);
+    }
 
     mPipelineCache.destroy(mDevice);
     mSamplerCache.destroy(this);
@@ -580,7 +585,7 @@ void RendererVk::notifyDeviceLost()
 {
     {
         std::lock_guard<std::mutex> lock(mQueueSerialMutex);
-        mLastCompletedQueueSerial = mLastSubmittedQueueSerial;
+        mLastCompletedQueueSerial = getLastSubmittedQueueSerial();
     }
     mDeviceLost = true;
     mDisplay->notifyDeviceLost();
@@ -909,7 +914,8 @@ angle::Result RendererVk::initialize(DisplayVk *displayVk,
     if (getFeatures().enableCommandProcessingThread.enabled)
     {
         mCommandProcessorThread =
-            std::thread(&CommandProcessor::processCommandProcessorTasks, &mCommandProcessor);
+            std::thread(&vk::CommandProcessor::processTasks, &mCommandProcessor);
+        mCommandProcessor.waitForWorkComplete(nullptr);
     }
 
     return angle::Result::Continue;
@@ -1907,6 +1913,9 @@ void RendererVk::initFeatures(DisplayVk *displayVk, const ExtensionNameList &dev
     // Currently disabled by default: http://anglebug.com/4324
     ANGLE_FEATURE_CONDITION(&mFeatures, enableCommandProcessingThread, false);
 
+    // Currently disabled by default: http://anglebug.com/4324
+    ANGLE_FEATURE_CONDITION(&mFeatures, asynchronousCommandProcessing, false);
+
     ANGLE_FEATURE_CONDITION(&mFeatures, supportsYUVSamplerConversion,
                             mSamplerYcbcrConversionFeatures.samplerYcbcrConversion != VK_FALSE);
 
@@ -2183,6 +2192,17 @@ bool RendererVk::hasBufferFormatFeatureBits(VkFormat format,
     return hasFormatFeatureBits<&VkFormatProperties::bufferFeatures>(format, featureBits);
 }
 
+void RendererVk::outputVmaStatString()
+{
+    // Output the VMA stats string
+    // This JSON string can be passed to VmaDumpVis.py to generate a visualization of the
+    // allocations the VMA has performed.
+    char *statsString;
+    mAllocator.buildStatsString(&statsString, true);
+    INFO() << std::endl << statsString << std::endl;
+    mAllocator.freeStatsString(statsString);
+}
+
 angle::Result RendererVk::queueSubmit(vk::Context *context,
                                       egl::ContextPriority priority,
                                       const VkSubmitInfo &submitInfo,
@@ -2192,23 +2212,11 @@ angle::Result RendererVk::queueSubmit(vk::Context *context,
 {
     if (kOutputVmaStatsString)
     {
-        // Output the VMA stats string
-        // This JSON string can be passed to VmaDumpVis.py to generate a visualization of the
-        // allocations the VMA has performed.
-        char *statsString;
-        mAllocator.buildStatsString(&statsString, true);
-        INFO() << std::endl << statsString << std::endl;
-        mAllocator.freeStatsString(statsString);
+        outputVmaStatString();
     }
 
-    if (getFeatures().enableCommandProcessingThread.enabled)
-    {
-        // For initial threading phase 1 code make sure any outstanding command processing
-        //  is complete.
-        // TODO: b/153666475 For phase2 investigate if this is required as most submits will take
-        //  place through worker thread except for one-off submits below.
-        mCommandProcessor.waitForWorkComplete();
-    }
+    ASSERT(!getFeatures().enableCommandProcessingThread.enabled);
+
     {
         std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
         std::lock_guard<std::mutex> serialLock(mQueueSerialMutex);
@@ -2235,12 +2243,26 @@ angle::Result RendererVk::queueSubmitOneOff(vk::Context *context,
                                             const vk::Fence *fence,
                                             Serial *serialOut)
 {
-    VkSubmitInfo submitInfo       = {};
-    submitInfo.sType              = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-    submitInfo.commandBufferCount = 1;
-    submitInfo.pCommandBuffers    = primary.ptr();
+    if (getFeatures().enableCommandProcessingThread.enabled)
+    {
+        vk::CommandProcessorTask oneOffQueueSubmit;
+        oneOffQueueSubmit.initOneOffQueueSubmit(primary.getHandle(), priority, fence);
+        queueCommand(context, &oneOffQueueSubmit);
+        waitForCommandProcessorIdle(context);
+        *serialOut = getLastSubmittedQueueSerial();
+
+        ANGLE_TRY(cleanupGarbage(false));
+    }
+    else
+    {
+
+        VkSubmitInfo submitInfo       = {};
+        submitInfo.sType              = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+        submitInfo.commandBufferCount = 1;
+        submitInfo.pCommandBuffers    = primary.ptr();
 
-    ANGLE_TRY(queueSubmit(context, priority, submitInfo, nullptr, fence, serialOut));
+        ANGLE_TRY(queueSubmit(context, priority, submitInfo, nullptr, fence, serialOut));
+    }
 
     mPendingOneOffCommands.push_back({*serialOut, std::move(primary)});
 
@@ -2251,8 +2273,8 @@ angle::Result RendererVk::queueWaitIdle(vk::Context *context, egl::ContextPriori
 {
     if (getFeatures().enableCommandProcessingThread.enabled)
     {
-        // First make sure command processor is complete when waiting for queue idle.
-        mCommandProcessor.waitForWorkComplete();
+        // Wait for all pending commands to get sent before issuing vkQueueWaitIdle
+        waitForCommandProcessorIdle(context);
     }
     {
         std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
@@ -2268,8 +2290,8 @@ angle::Result RendererVk::deviceWaitIdle(vk::Context *context)
 {
     if (getFeatures().enableCommandProcessingThread.enabled)
     {
-        // First make sure command processor is complete when waiting for device idle.
-        mCommandProcessor.waitForWorkComplete();
+        // Wait for all pending commands to get sent before issuing vkQueueWaitIdle
+        waitForCommandProcessorIdle(context);
     }
     {
         std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
@@ -2286,12 +2308,7 @@ VkResult RendererVk::queuePresent(egl::ContextPriority priority,
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::queuePresent");
 
-    if (getFeatures().enableCommandProcessingThread.enabled)
-    {
-        // First make sure command processor is complete before queue present as
-        //  present may have dependencies on that thread.
-        mCommandProcessor.waitForWorkComplete();
-    }
+    ASSERT(!getFeatures().enableCommandProcessingThread.enabled);
 
     std::lock_guard<decltype(mQueueMutex)> lock(mQueueMutex);
 
@@ -2329,6 +2346,30 @@ angle::Result RendererVk::newSharedFence(vk::Context *context,
     return angle::Result::Continue;
 }
 
+// Return a shared fence to be used for the next submit
+// Fence may be shared with a Sync object.
+// reset indicates that nextSubmitFence should be reset before returning. This ensures that the next
+// request for a submit fence gets a fresh fence.
+// TODO: https://issuetracker.google.com/issues/170312581 - move to CommandProcessor as part of
+// fence ownership follow-up task.
+angle::Result RendererVk::getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOut, bool reset)
+{
+    std::lock_guard<decltype(mNextSubmitFenceMutex)> lock(mNextSubmitFenceMutex);
+    if (!mNextSubmitFence.isReferenced())
+    {
+        ANGLE_TRY(newSharedFence(&mCommandProcessor, &mNextSubmitFence));
+    }
+
+    ASSERT(!sharedFenceOut->isReferenced());
+    sharedFenceOut->copy(getDevice(), mNextSubmitFence);
+
+    if (reset)
+    {
+        resetSharedFence(&mNextSubmitFence);
+    }
+    return angle::Result::Continue;
+}
+
 template <VkFormatFeatureFlags VkFormatProperties::*features>
 VkFormatFeatureFlags RendererVk::getFormatFeatureBits(VkFormat format,
                                                       const VkFormatFeatureFlags featureBits) const

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -168,6 +168,7 @@ class RendererVk : angle::NonCopyable
         return mPriorities[priority];
     }
 
+    // Queue submit that originates from the main thread
     angle::Result queueSubmit(vk::Context *context,
                               egl::ContextPriority priority,
                               const VkSubmitInfo &submitInfo,
@@ -197,6 +198,8 @@ class RendererVk : angle::NonCopyable
         sharedFenceIn->resetAndRecycle(&mFenceRecycler);
     }
 
+    angle::Result getNextSubmitFence(vk::Shared<vk::Fence> *sharedFenceOut, bool reset);
+
     template <typename... ArgsT>
     void collectGarbageAndReinit(vk::SharedResourceUse *use, ArgsT... garbageIn)
     {
@@ -224,6 +227,12 @@ class RendererVk : angle::NonCopyable
         }
     }
 
+    vk::Shared<vk::Fence> getLastSubmittedFence() const
+    {
+        return mCommandProcessor.getLastSubmittedFence();
+    }
+    void handleDeviceLost() { mCommandProcessor.handleDeviceLost(); }
+
     static constexpr size_t kMaxExtensionNames = 200;
     using ExtensionNameList = angle::FixedVector<const char *, kMaxExtensionNames>;
 
@@ -241,11 +250,19 @@ class RendererVk : angle::NonCopyable
 
     ANGLE_INLINE Serial getCurrentQueueSerial()
     {
+        if (getFeatures().enableCommandProcessingThread.enabled)
+        {
+            return mCommandProcessor.getCurrentQueueSerial();
+        }
         std::lock_guard<std::mutex> lock(mQueueSerialMutex);
         return mCurrentQueueSerial;
     }
     ANGLE_INLINE Serial getLastSubmittedQueueSerial()
     {
+        if (getFeatures().enableCommandProcessingThread.enabled)
+        {
+            return mCommandProcessor.getLastSubmittedSerial();
+        }
         std::lock_guard<std::mutex> lock(mQueueSerialMutex);
         return mLastSubmittedQueueSerial;
     }
@@ -264,11 +281,24 @@ class RendererVk : angle::NonCopyable
     vk::ActiveHandleCounter &getActiveHandleCounts() { return mActiveHandleCounts; }
 
     // Queue commands to worker thread for processing
-    void queueCommands(const vk::CommandProcessorTask &commands)
+    void queueCommand(vk::Context *context, vk::CommandProcessorTask *command)
     {
-        mCommandProcessor.queueCommands(commands);
+        mCommandProcessor.queueCommand(context, command);
+    }
+    bool hasPendingError() const { return mCommandProcessor.hasPendingError(); }
+    vk::Error getAndClearPendingError() { return mCommandProcessor.getAndClearPendingError(); }
+    void waitForCommandProcessorIdle(vk::Context *context)
+    {
+        mCommandProcessor.waitForWorkComplete(context);
     }
-    void waitForWorkerThreadIdle() { mCommandProcessor.waitForWorkComplete(); }
+
+    void finishToSerial(vk::Context *context, Serial serial)
+    {
+        mCommandProcessor.finishToSerial(context, serial);
+    }
+
+    void finishAllWork(vk::Context *context) { mCommandProcessor.finishAllWork(context); }
+    VkQueue getVkQueue(egl::ContextPriority priority) const { return mQueues[priority]; }
 
     bool getEnableValidationLayers() const { return mEnableValidationLayers; }
 
@@ -276,6 +306,8 @@ class RendererVk : angle::NonCopyable
 
     void setGlobalDebugAnnotator();
 
+    void outputVmaStatString();
+
   private:
     angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
@@ -390,9 +422,13 @@ class RendererVk : angle::NonCopyable
     };
     std::deque<PendingOneOffCommands> mPendingOneOffCommands;
 
-    // Worker Thread
-    CommandProcessor mCommandProcessor;
+    // Command Processor Thread
+    vk::CommandProcessor mCommandProcessor;
     std::thread mCommandProcessorThread;
+    // mNextSubmitFence is the fence that's going to be signaled at the next submission.  This is
+    // used to support SyncVk objects, which may outlive the context (as EGLSync objects).
+    vk::Shared<vk::Fence> mNextSubmitFence;
+    std::mutex mNextSubmitFenceMutex;
 
     // track whether we initialized (or released) glslang
     bool mGlslangInitialized;

src/libANGLE/renderer/vulkan/ResourceVk.cpp

@@ -102,11 +102,22 @@ ResourceUseList::ResourceUseList()
     mResourceUses.reserve(kDefaultResourceUseCount);
 }
 
+ResourceUseList::ResourceUseList(ResourceUseList &&other)
+{
+    *this = std::move(other);
+}
+
 ResourceUseList::~ResourceUseList()
 {
     ASSERT(mResourceUses.empty());
 }
 
+ResourceUseList &ResourceUseList::operator=(ResourceUseList &&rhs)
+{
+    std::swap(mResourceUses, rhs.mResourceUses);
+    return *this;
+}
+
 void ResourceUseList::releaseResourceUses()
 {
     for (SharedResourceUse &use : mResourceUses)

src/libANGLE/renderer/vulkan/ResourceVk.h

@@ -136,7 +136,9 @@ class ResourceUseList final : angle::NonCopyable
 {
   public:
     ResourceUseList();
+    ResourceUseList(ResourceUseList &&other);
     virtual ~ResourceUseList();
+    ResourceUseList &operator=(ResourceUseList &&rhs);
 
     void add(const SharedResourceUse &resourceUse);
 

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -1186,6 +1186,7 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
                                        bool *presentOutOfDate)
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "WindowSurfaceVk::present");
+    RendererVk *renderer = contextVk->getRenderer();
 
     // Throttle the submissions to avoid getting too far ahead of the GPU.
     SwapHistory &swap = mSwapHistory[mCurrentSwapHistoryIndex];
@@ -1194,7 +1195,7 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
         if (swap.sharedFence.isReferenced())
         {
             ANGLE_TRY(swap.waitFence(contextVk));
-            swap.destroy(contextVk->getRenderer());
+            swap.destroy(renderer);
         }
     }
 
@@ -1316,6 +1317,8 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
     }
 
     // Update the swap history for this presentation
+    // TODO: https://issuetracker.google.com/issues/170312581 - this will force us to flush worker
+    // queue to get the fence.
     swap.sharedFence = contextVk->getLastSubmittedFence();
     ASSERT(!mAcquireImageSemaphore.valid());
 
@@ -1323,13 +1326,47 @@ angle::Result WindowSurfaceVk::present(ContextVk *contextVk,
     mCurrentSwapHistoryIndex =
         mCurrentSwapHistoryIndex == mSwapHistory.size() ? 0 : mCurrentSwapHistoryIndex;
 
-    VkResult result = contextVk->getRenderer()->queuePresent(contextVk->getPriority(), presentInfo);
+    VkResult result;
+    if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+    {
+        vk::CommandProcessorTask present;
+        present.initPresent(contextVk->getPriority(), presentInfo);
+
+        // Make sure everything has been submitted (and errors handled)
+        renderer->waitForCommandProcessorIdle(contextVk);
+
+        // Submit queuePresent all by itself (ignoring interference from other threads for now)
+        renderer->queueCommand(contextVk, &present);
+        // TODO: https://issuetracker.google.com/issues/170329600 - Just stalling here for now, but
+        // really want to let main thread continue
+        // need to figure out how to handle work below off-thread and sync to main
+        // Also, need to fix lifetime of presentInfo data when main thread continues.
+        // There is a bunch of work happening after present to deal with swapchain recreation.
+        // Will that require moving a large chunk of swapImpl to the CommandProcessor?
+        // That will likely require serializing access to the WindowSurfaceVk object in order
+        // to have current content.
+        result = VK_SUCCESS;
+
+        // wait for the queuePresent to be submitted and intentionally set the context to nullptr so
+        // that we can catch any error. Note this doesn't prevent another context from grabbing the
+        // error. Will be fixed properly in a follow-up as part of present work.
+        renderer->waitForCommandProcessorIdle(nullptr);
+        if (renderer->hasPendingError())
+        {
+            vk::Error error = renderer->getAndClearPendingError();
+            result          = error.mErrorCode;
+        }
+    }
+    else
+    {
+        result = renderer->queuePresent(contextVk->getPriority(), presentInfo);
+    }
 
     // If OUT_OF_DATE is returned, it's ok, we just need to recreate the swapchain before
     // continuing.
     // If VK_SUBOPTIMAL_KHR is returned it's because the device orientation changed and we should
     // recreate the swapchain with a new window orientation.
-    if (contextVk->getFeatures().enablePreRotateSurfaces.enabled)
+    if (renderer->getFeatures().enablePreRotateSurfaces.enabled)
     {
         // Also check for VK_SUBOPTIMAL_KHR.
         *presentOutOfDate = ((result == VK_ERROR_OUT_OF_DATE_KHR) || (result == VK_SUBOPTIMAL_KHR));

src/libANGLE/renderer/vulkan/SyncVk.cpp

@@ -32,6 +32,12 @@ SyncHelper::~SyncHelper() {}
 void SyncHelper::releaseToRenderer(RendererVk *renderer)
 {
     renderer->collectGarbageAndReinit(&mUse, &mEvent);
+    // TODO: https://issuetracker.google.com/170312581 - Currently just stalling on worker thread
+    // here to try and avoid race condition. If this works, need some alternate solution
+    if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+    {
+        renderer->waitForCommandProcessorIdle(nullptr);
+    }
     mFence.reset(renderer->getDevice());
 }
 
@@ -48,7 +54,17 @@ angle::Result SyncHelper::initialize(ContextVk *contextVk)
 
     DeviceScoped<Event> event(device);
     ANGLE_VK_TRY(contextVk, event.get().init(device, eventCreateInfo));
-    ANGLE_TRY(contextVk->getNextSubmitFence(&mFence));
+    // TODO: https://issuetracker.google.com/170312581 - For now wait for worker thread to finish
+    // then get next fence from renderer
+    if (contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+    {
+        contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
+        ANGLE_TRY(contextVk->getRenderer()->getNextSubmitFence(&mFence, false));
+    }
+    else
+    {
+        ANGLE_TRY(contextVk->getNextSubmitFence(&mFence));
+    }
 
     mEvent = event.release();
 
@@ -90,9 +106,17 @@ angle::Result SyncHelper::clientWait(Context *context,
         ANGLE_TRY(contextVk->flushImpl(nullptr));
     }
 
+    // If we are using worker need to wait for the commands to be issued before waiting on the
+    // fence.
+    if (contextVk->getRenderer()->getFeatures().enableCommandProcessingThread.enabled)
+    {
+        contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
+    }
+
     // Wait on the fence that's expected to be signaled on the first vkQueueSubmit after
     // `initialize` was called. The first fence is the fence created to signal this sync.
     ASSERT(mFence.get().valid());
+    // TODO: https://issuetracker.google.com/170312581 - Wait could be command to worker
     VkResult status = mFence.get().wait(renderer->getDevice(), timeout);
 
     // Check for errors, but don't consider timeout as such.
@@ -189,11 +213,24 @@ angle::Result SyncHelperNativeFence::initializeWithFd(ContextVk *contextVk, int
 
         retain(&contextVk->getResourceUseList());
 
-        Serial serialOut;
-        VkSubmitInfo submitInfo = {};
-        submitInfo.sType        = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-        ANGLE_TRY(renderer->queueSubmit(contextVk, contextVk->getPriority(), submitInfo, nullptr,
-                                        &fence.get(), &serialOut));
+        if (renderer->getFeatures().enableCommandProcessingThread.enabled)
+        {
+            CommandProcessorTask oneOffQueueSubmit;
+            oneOffQueueSubmit.initOneOffQueueSubmit(VK_NULL_HANDLE, contextVk->getPriority(),
+                                                    &fence.get());
+            renderer->queueCommand(contextVk, &oneOffQueueSubmit);
+            // TODO: https://issuetracker.google.com/170312581 - wait for now
+            renderer->waitForCommandProcessorIdle(contextVk);
+        }
+        else
+        {
+            Serial serialOut;
+            VkSubmitInfo submitInfo = {};
+            submitInfo.sType        = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+
+            ANGLE_TRY(renderer->queueSubmit(contextVk, contextVk->getPriority(), submitInfo,
+                                            nullptr, &fence.get(), &serialOut));
+        }
 
         VkFenceGetFdInfoKHR fenceGetFdInfo = {};
         fenceGetFdInfo.sType               = VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR;
@@ -253,6 +290,14 @@ angle::Result SyncHelperNativeFence::clientWait(Context *context,
     {
         ANGLE_TRY(contextVk->flushImpl(nullptr));
     }
+
+    // If we are using worker need to wait for the commands to be issued before waiting on the
+    // fence.
+    if (contextVk->getRenderer()->getFeatures().asynchronousCommandProcessing.enabled)
+    {
+        contextVk->getRenderer()->waitForCommandProcessorIdle(contextVk);
+    }
+
     // Wait for mFenceWithFd to be signaled.
     VkResult status = mFenceWithFd.wait(renderer->getDevice(), timeout);
 

src/libANGLE/renderer/vulkan/UtilsVk.cpp

@@ -969,6 +969,7 @@ angle::Result UtilsVk::setupProgram(ContextVk *contextVk,
         vk::PipelineAndSerial *pipelineAndSerial;
         program->setShader(gl::ShaderType::Compute, fsCsShader);
         ANGLE_TRY(program->getComputePipeline(contextVk, pipelineLayout.get(), &pipelineAndSerial));
+        // TODO: https://issuetracker.google.com/issues/169788986: Update serial handling.
         pipelineAndSerial->updateSerial(serial);
         commandBuffer->bindComputePipeline(pipelineAndSerial->get());
     }

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -916,7 +916,8 @@ void CommandBufferHelper::restoreStencilContent()
     }
 }
 
-void CommandBufferHelper::executeBarriers(ContextVk *contextVk, PrimaryCommandBuffer *primary)
+void CommandBufferHelper::executeBarriers(const angle::FeaturesVk &features,
+                                          PrimaryCommandBuffer *primary)
 {
     // make a local copy for faster access
     PipelineStagesMask mask = mPipelineBarrierMask;
@@ -925,7 +926,7 @@ void CommandBufferHelper::executeBarriers(ContextVk *contextVk, PrimaryCommandBu
         return;
     }
 
-    if (contextVk->getFeatures().preferAggregateBarrierCalls.enabled)
+    if (features.preferAggregateBarrierCalls.enabled)
     {
         PipelineStagesMask::Iterator iter = mask.begin();
         PipelineBarrier &barrier          = mPipelineBarriers[*iter];
@@ -1152,24 +1153,31 @@ void CommandBufferHelper::endTransformFeedback()
     mValidTransformFeedbackBufferCount = 0;
 }
 
-angle::Result CommandBufferHelper::flushToPrimary(ContextVk *contextVk,
-                                                  PrimaryCommandBuffer *primary)
+angle::Result CommandBufferHelper::getRenderPassWithOps(ContextVk *contextVk,
+                                                        RenderPass **renderPass)
 {
-    ANGLE_TRACE_EVENT0("gpu.angle", "CommandBufferHelper::flushToPrimary");
-    ASSERT(!empty());
-    if (kEnableCommandStreamDiagnostics)
+    *renderPass = nullptr;
+    if (mIsRenderPassCommandBuffer)
     {
-        addCommandDiagnostics(contextVk);
+        ANGLE_TRY(contextVk->getRenderPassWithOps(mRenderPassDesc, mAttachmentOps, renderPass));
     }
+    return angle::Result::Continue;
+}
+
+angle::Result CommandBufferHelper::flushToPrimary(const angle::FeaturesVk &features,
+                                                  PrimaryCommandBuffer *primary,
+                                                  RenderPass *renderPass)
+{
+    ANGLE_TRACE_EVENT0("gpu.angle", "CommandBufferHelper::flushToPrimary");
+    ASSERT(!empty());
+
     // Commands that are added to primary before beginRenderPass command
-    executeBarriers(contextVk, primary);
+    executeBarriers(features, primary);
 
     if (mIsRenderPassCommandBuffer)
     {
         mCommandBuffer.executeQueuedResetQueryPoolCommands(primary->getHandle());
-        // Pull a RenderPass from the cache.
-        RenderPass *renderPass = nullptr;
-        ANGLE_TRY(contextVk->getRenderPassWithOps(mRenderPassDesc, mAttachmentOps, &renderPass));
+        ASSERT(renderPass != nullptr);
 
         VkRenderPassBeginInfo beginInfo    = {};
         beginInfo.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;

src/libANGLE/renderer/vulkan/vk_helpers.h

@@ -962,9 +962,12 @@ class CommandBufferHelper : angle::NonCopyable
 
     CommandBuffer &getCommandBuffer() { return mCommandBuffer; }
 
-    angle::Result flushToPrimary(ContextVk *contextVk, PrimaryCommandBuffer *primary);
+    angle::Result getRenderPassWithOps(ContextVk *contextVk, RenderPass **renderPass);
+    angle::Result flushToPrimary(const angle::FeaturesVk &features,
+                                 PrimaryCommandBuffer *primary,
+                                 RenderPass *renderPass);
 
-    void executeBarriers(ContextVk *contextVk, PrimaryCommandBuffer *primary);
+    void executeBarriers(const angle::FeaturesVk &features, PrimaryCommandBuffer *primary);
 
     void setHasRenderPass(bool hasRenderPass) { mIsRenderPassCommandBuffer = hasRenderPass; }
 
@@ -1129,9 +1132,9 @@ class CommandBufferHelper : angle::NonCopyable
 
     bool isReadOnlyDepthMode() const { return mReadOnlyDepthStencilMode; }
 
-  private:
     void addCommandDiagnostics(ContextVk *contextVk);
 
+  private:
     bool onDepthStencilAccess(ResourceAccess access,
                               uint32_t *cmdCountInvalidated,
                               uint32_t *cmdCountDisabled);

src/libANGLE/renderer/vulkan/vk_utils.h

@@ -943,15 +943,15 @@ gl::LevelIndex GetLevelIndex(vk::LevelIndex levelVk, gl::LevelIndex baseLevel);
 
 }  // namespace rx
 
-#define ANGLE_VK_TRY(context, command)                                                 \
-    do                                                                                 \
-    {                                                                                  \
-        auto ANGLE_LOCAL_VAR = command;                                                \
-        if (ANGLE_UNLIKELY(ANGLE_LOCAL_VAR != VK_SUCCESS))                             \
-        {                                                                              \
-            context->handleError(ANGLE_LOCAL_VAR, __FILE__, ANGLE_FUNCTION, __LINE__); \
-            return angle::Result::Stop;                                                \
-        }                                                                              \
+#define ANGLE_VK_TRY(context, command)                                                   \
+    do                                                                                   \
+    {                                                                                    \
+        auto ANGLE_LOCAL_VAR = command;                                                  \
+        if (ANGLE_UNLIKELY(ANGLE_LOCAL_VAR != VK_SUCCESS))                               \
+        {                                                                                \
+            (context)->handleError(ANGLE_LOCAL_VAR, __FILE__, ANGLE_FUNCTION, __LINE__); \
+            return angle::Result::Stop;                                                  \
+        }                                                                                \
     } while (0)
 
 #define ANGLE_VK_CHECK(context, test, error) ANGLE_VK_TRY(context, test ? VK_SUCCESS : error)