Vulkan: Implement command re-ordering.

src/libANGLE/renderer/vulkan/BufferVk.cpp

@@ -191,11 +191,8 @@ vk::Error BufferVk::setDataImpl(ContextVk *contextVk,
         stagingBuffer.getDeviceMemory().unmap(device);
 
         // Enqueue a copy command on the GPU.
-        // TODO(jmadill): Command re-ordering for render passes.
-        renderer->endRenderPass();
-
-        vk::CommandBufferAndState *commandBuffer = nullptr;
-        ANGLE_TRY(renderer->getStartedCommandBuffer(&commandBuffer));
+        vk::CommandBuffer *commandBuffer = nullptr;
+        ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
         // Insert a barrier to ensure reads from the buffer are complete.
         // TODO(jmadill): Insert minimal barriers.
@@ -216,7 +213,8 @@ vk::Error BufferVk::setDataImpl(ContextVk *contextVk,
         VkBufferCopy copyRegion = {offset, 0, size};
         commandBuffer->copyBuffer(stagingBuffer.getBuffer(), mBuffer, 1, &copyRegion);
 
-        setQueueSerial(renderer->getCurrentQueueSerial());
+        // Immediately release staging buffer.
+        // TODO(jmadill): Staging buffer re-use.
         renderer->releaseObject(getQueueSerial(), &stagingBuffer);
     }
     else

src/libANGLE/renderer/vulkan/CommandBufferNode.cpp

+//
+// Copyright 2017 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// CommandBufferNode:
+//    Deferred work constructed by GL calls, that will later be flushed to Vulkan.
+//
+
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
+
+#include "libANGLE/renderer/vulkan/RenderTargetVk.h"
+#include "libANGLE/renderer/vulkan/RendererVk.h"
+#include "libANGLE/renderer/vulkan/formatutilsvk.h"
+
+namespace rx
+{
+
+namespace vk
+{
+
+namespace
+{
+
+Error InitAndBeginCommandBuffer(VkDevice device,
+                                const CommandPool &commandPool,
+                                const VkCommandBufferInheritanceInfo &inheritanceInfo,
+                                VkCommandBufferUsageFlags flags,
+                                CommandBuffer *commandBuffer)
+{
+    ASSERT(!commandBuffer->valid());
+
+    VkCommandBufferAllocateInfo createInfo;
+    createInfo.sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+    createInfo.pNext              = nullptr;
+    createInfo.commandPool        = commandPool.getHandle();
+    createInfo.level              = VK_COMMAND_BUFFER_LEVEL_SECONDARY;
+    createInfo.commandBufferCount = 1;
+
+    ANGLE_TRY(commandBuffer->init(device, createInfo));
+
+    VkCommandBufferBeginInfo beginInfo;
+    beginInfo.sType            = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+    beginInfo.pNext            = nullptr;
+    beginInfo.flags            = flags | VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+    beginInfo.pInheritanceInfo = &inheritanceInfo;
+
+    ANGLE_TRY(commandBuffer->begin(beginInfo));
+    return NoError();
+}
+
+}  // anonymous namespace
+
+// CommandBufferNode implementation.
+
+CommandBufferNode::CommandBufferNode()
+    : mIsDependency(false), mVisitedState(VisitedState::Unvisited)
+{
+}
+
+CommandBufferNode::~CommandBufferNode()
+{
+    mRenderPassFramebuffer.setHandle(VK_NULL_HANDLE);
+
+    // Command buffers are managed by the command pool, so don't need to be freed.
+    mOutsideRenderPassCommands.releaseHandle();
+    mInsideRenderPassCommands.releaseHandle();
+}
+
+CommandBuffer *CommandBufferNode::getOutsideRenderPassCommands()
+{
+    return &mOutsideRenderPassCommands;
+}
+
+CommandBuffer *CommandBufferNode::getInsideRenderPassCommands()
+{
+    return &mInsideRenderPassCommands;
+}
+
+Error CommandBufferNode::startRecording(VkDevice device,
+                                        const CommandPool &commandPool,
+                                        CommandBuffer **commandsOut)
+{
+    VkCommandBufferInheritanceInfo inheritanceInfo;
+    inheritanceInfo.sType                = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
+    inheritanceInfo.pNext                = nullptr;
+    inheritanceInfo.renderPass           = VK_NULL_HANDLE;
+    inheritanceInfo.subpass              = 0;
+    inheritanceInfo.framebuffer          = VK_NULL_HANDLE;
+    inheritanceInfo.occlusionQueryEnable = VK_FALSE;
+    inheritanceInfo.queryFlags           = 0;
+    inheritanceInfo.pipelineStatistics   = 0;
+
+    ANGLE_TRY(InitAndBeginCommandBuffer(device, commandPool, inheritanceInfo, 0,
+                                        &mOutsideRenderPassCommands));
+
+    *commandsOut = &mOutsideRenderPassCommands;
+    return NoError();
+}
+
+Error CommandBufferNode::startRenderPassRecording(RendererVk *renderer, CommandBuffer **commandsOut)
+{
+    // Get a compatible RenderPass from the cache so we can initialize the inheritance info.
+    // TODO(jmadill): Use different query method for compatible vs conformant render pass.
+    vk::RenderPass *compatibleRenderPass;
+    ANGLE_TRY(renderer->getCompatibleRenderPass(mRenderPassDesc, &compatibleRenderPass));
+
+    VkCommandBufferInheritanceInfo inheritanceInfo;
+    inheritanceInfo.sType                = VK_STRUCTURE_TYPE_COMMAND_BUFFER_INHERITANCE_INFO;
+    inheritanceInfo.pNext                = nullptr;
+    inheritanceInfo.renderPass           = compatibleRenderPass->getHandle();
+    inheritanceInfo.subpass              = 0;
+    inheritanceInfo.framebuffer          = mRenderPassFramebuffer.getHandle();
+    inheritanceInfo.occlusionQueryEnable = VK_FALSE;
+    inheritanceInfo.queryFlags           = 0;
+    inheritanceInfo.pipelineStatistics   = 0;
+
+    ANGLE_TRY(InitAndBeginCommandBuffer(
+        renderer->getDevice(), renderer->getCommandPool(), inheritanceInfo,
+        VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT, &mInsideRenderPassCommands));
+
+    *commandsOut = &mInsideRenderPassCommands;
+    return NoError();
+}
+
+void CommandBufferNode::storeRenderPassInfo(const Framebuffer &framebuffer,
+                                            const gl::Rectangle renderArea,
+                                            const std::vector<VkClearValue> &clearValues)
+{
+    mRenderPassFramebuffer.setHandle(framebuffer.getHandle());
+    mRenderPassRenderArea = renderArea;
+    std::copy(clearValues.begin(), clearValues.end(), mRenderPassClearValues.begin());
+}
+
+void CommandBufferNode::appendColorRenderTarget(Serial serial, RenderTargetVk *colorRenderTarget)
+{
+    // TODO(jmadill): Layout transition?
+    mRenderPassDesc.packColorAttachment(*colorRenderTarget->format, colorRenderTarget->samples);
+    colorRenderTarget->resource->setWriteNode(serial, this);
+}
+
+void CommandBufferNode::appendDepthStencilRenderTarget(Serial serial,
+                                                       RenderTargetVk *depthStencilRenderTarget)
+{
+    // TODO(jmadill): Layout transition?
+    mRenderPassDesc.packDepthStencilAttachment(*depthStencilRenderTarget->format,
+                                               depthStencilRenderTarget->samples);
+    depthStencilRenderTarget->resource->setWriteNode(serial, this);
+}
+
+void CommandBufferNode::initAttachmentDesc(VkAttachmentDescription *desc)
+{
+    desc->flags          = 0;
+    desc->format         = VK_FORMAT_UNDEFINED;
+    desc->samples        = static_cast<VkSampleCountFlagBits>(0);
+    desc->loadOp         = VK_ATTACHMENT_LOAD_OP_CLEAR;
+    desc->storeOp        = VK_ATTACHMENT_STORE_OP_STORE;
+    desc->stencilLoadOp  = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+    desc->stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
+    desc->initialLayout  = VK_IMAGE_LAYOUT_UNDEFINED;
+    desc->finalLayout    = VK_IMAGE_LAYOUT_UNDEFINED;
+}
+
+void CommandBufferNode::addDependency(CommandBufferNode *node)
+{
+    mDependencies.emplace_back(node);
+    node->markAsDependency();
+    ASSERT(node != this && !node->hasDependency(this));
+}
+
+void CommandBufferNode::addDependencies(const std::vector<CommandBufferNode *> &nodes)
+{
+    mDependencies.insert(mDependencies.end(), nodes.begin(), nodes.end());
+
+    // TODO(jmadill): is there a faster way to do this?
+    for (CommandBufferNode *node : nodes)
+    {
+        node->markAsDependency();
+        ASSERT(node != this && !node->hasDependency(this));
+    }
+}
+
+bool CommandBufferNode::hasDependencies() const
+{
+    return !mDependencies.empty();
+}
+
+void CommandBufferNode::markAsDependency()
+{
+    mIsDependency = true;
+}
+
+bool CommandBufferNode::isDependency() const
+{
+    return mIsDependency;
+}
+
+// Do not call this in anything but testing code, since it's slow.
+bool CommandBufferNode::hasDependency(CommandBufferNode *searchNode)
+{
+    std::set<CommandBufferNode *> visitedList;
+    std::vector<CommandBufferNode *> openList;
+    openList.insert(openList.begin(), mDependencies.begin(), mDependencies.end());
+    while (!openList.empty())
+    {
+        CommandBufferNode *node = openList.back();
+        openList.pop_back();
+        if (visitedList.count(node) == 0)
+        {
+            if (node == searchNode)
+            {
+                return true;
+            }
+            visitedList.insert(node);
+            openList.insert(openList.end(), node->mDependencies.begin(), node->mDependencies.end());
+        }
+    }
+
+    return false;
+}
+
+VisitedState CommandBufferNode::visitedState() const
+{
+    return mVisitedState;
+}
+
+void CommandBufferNode::visitDependencies(std::vector<CommandBufferNode *> *stack)
+{
+    ASSERT(mVisitedState == VisitedState::Unvisited);
+    stack->insert(stack->end(), mDependencies.begin(), mDependencies.end());
+    mVisitedState = VisitedState::Ready;
+}
+
+vk::Error CommandBufferNode::visitAndExecute(RendererVk *renderer,
+                                             vk::CommandBuffer *primaryCommandBuffer)
+{
+    if (mOutsideRenderPassCommands.valid())
+    {
+        mOutsideRenderPassCommands.end();
+        primaryCommandBuffer->executeCommands(1, &mOutsideRenderPassCommands);
+    }
+
+    if (mInsideRenderPassCommands.valid())
+    {
+        // Pull a compatible RenderPass from the cache.
+        // TODO(jmadill): Insert real ops and layout transitions.
+        vk::RenderPass *renderPass = nullptr;
+        ANGLE_TRY(renderer->getCompatibleRenderPass(mRenderPassDesc, &renderPass));
+
+        mInsideRenderPassCommands.end();
+
+        VkRenderPassBeginInfo beginInfo;
+        beginInfo.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
+        beginInfo.pNext                    = nullptr;
+        beginInfo.renderPass               = renderPass->getHandle();
+        beginInfo.framebuffer              = mRenderPassFramebuffer.getHandle();
+        beginInfo.renderArea.offset.x      = static_cast<uint32_t>(mRenderPassRenderArea.x);
+        beginInfo.renderArea.offset.y      = static_cast<uint32_t>(mRenderPassRenderArea.y);
+        beginInfo.renderArea.extent.width  = static_cast<uint32_t>(mRenderPassRenderArea.width);
+        beginInfo.renderArea.extent.height = static_cast<uint32_t>(mRenderPassRenderArea.height);
+        beginInfo.clearValueCount          = mRenderPassDesc.attachmentCount();
+        beginInfo.pClearValues             = mRenderPassClearValues.data();
+
+        primaryCommandBuffer->beginRenderPass(beginInfo,
+                                              VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
+        primaryCommandBuffer->executeCommands(1, &mInsideRenderPassCommands);
+        primaryCommandBuffer->endRenderPass();
+    }
+
+    mVisitedState = VisitedState::Visited;
+    return vk::NoError();
+}
+
+}  // namespace vk
+}  // namespace rx

src/libANGLE/renderer/vulkan/CommandBufferNode.h

+//
+// Copyright 2017 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// CommandBufferNode:
+//    Deferred work constructed by GL calls, that will later be flushed to Vulkan.
+//
+
+#ifndef LIBANGLE_RENDERER_VULKAN_COMMAND_BUFFER_NODE_H_
+#define LIBANGLE_RENDERER_VULKAN_COMMAND_BUFFER_NODE_H_
+
+#include "libANGLE/renderer/vulkan/renderervk_utils.h"
+
+namespace rx
+{
+
+namespace vk
+{
+
+enum class VisitedState
+{
+    Unvisited,
+    Ready,
+    Visited,
+};
+
+class CommandBufferNode final : angle::NonCopyable
+{
+  public:
+    CommandBufferNode();
+    ~CommandBufferNode();
+
+    // Immutable queries for when we're walking the commands tree.
+    CommandBuffer *getOutsideRenderPassCommands();
+    CommandBuffer *getInsideRenderPassCommands();
+
+    // For outside the render pass (copies, transitions, etc).
+    Error startRecording(VkDevice device,
+                         const CommandPool &commandPool,
+                         CommandBuffer **commandsOut);
+
+    // For rendering commands (draws).
+    Error startRenderPassRecording(RendererVk *renderer, CommandBuffer **commandsOut);
+
+    // Commands for storing info relevant to the RenderPass.
+    // RenderTargets must be added in order, with the depth/stencil being added last.
+    void storeRenderPassInfo(const Framebuffer &framebuffer,
+                             const gl::Rectangle renderArea,
+                             const std::vector<VkClearValue> &clearValues);
+    void appendColorRenderTarget(Serial serial, RenderTargetVk *colorRenderTarget);
+    void appendDepthStencilRenderTarget(Serial serial, RenderTargetVk *depthStencilRenderTarget);
+
+    // Commands for linking nodes in the dependency graph.
+    void addDependency(CommandBufferNode *node);
+    void addDependencies(const std::vector<CommandBufferNode *> &nodes);
+    bool hasDependencies() const;
+    bool isDependency() const;
+
+    // Used for testing only.
+    bool hasDependency(CommandBufferNode *searchNode);
+
+    // Commands for traversing the node on a flush operation.
+    VisitedState visitedState() const;
+    void visitDependencies(std::vector<CommandBufferNode *> *stack);
+    Error visitAndExecute(RendererVk *renderer, CommandBuffer *primaryCommandBuffer);
+
+  private:
+    void initAttachmentDesc(VkAttachmentDescription *desc);
+    void markAsDependency();
+
+    // Only used if we need a RenderPass for these commands.
+    RenderPassDesc mRenderPassDesc;
+    Framebuffer mRenderPassFramebuffer;
+    gl::Rectangle mRenderPassRenderArea;
+    gl::AttachmentArray<VkClearValue> mRenderPassClearValues;
+
+    // Keep a separate buffers for commands inside and outside a RenderPass.
+    // TODO(jmadill): We might not need inside and outside RenderPass commands separate.
+    CommandBuffer mOutsideRenderPassCommands;
+    CommandBuffer mInsideRenderPassCommands;
+
+    // Dependency commands must finish before these command can execute.
+    std::vector<CommandBufferNode *> mDependencies;
+    bool mIsDependency;
+
+    // Used when traversing the dependency graph.
+    VisitedState mVisitedState;
+};
+}  // namespace vk
+}  // namespace rx
+
+#endif  // LIBANGLE_RENDERER_VULKAN_COMMAND_BUFFER_NODE_H_

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -14,6 +14,7 @@
 #include "libANGLE/Context.h"
 #include "libANGLE/Program.h"
 #include "libANGLE/renderer/vulkan/BufferVk.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/CompilerVk.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
 #include "libANGLE/renderer/vulkan/DeviceVk.h"
@@ -62,7 +63,11 @@ enum DescriptorPoolIndex : uint8_t
 }  // anonymous namespace
 
 ContextVk::ContextVk(const gl::ContextState &state, RendererVk *renderer)
-    : ContextImpl(state), mRenderer(renderer), mCurrentDrawMode(GL_NONE)
+    : ContextImpl(state),
+      mRenderer(renderer),
+      mCurrentDrawMode(GL_NONE),
+      mVertexArrayDirty(false),
+      mTexturesDirty(false)
 {
     // The module handle is filled out at draw time.
     mCurrentShaderStages[0].sType  = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
@@ -238,15 +243,14 @@ gl::Error ContextVk::initialize()
 
 gl::Error ContextVk::flush(const gl::Context *context)
 {
+    // TODO(jmadill): Flush will need to insert a semaphore for the next flush to wait on.
     UNIMPLEMENTED();
     return gl::InternalError();
 }
 
 gl::Error ContextVk::finish(const gl::Context *context)
 {
-    // TODO(jmadill): Implement finish.
-    // UNIMPLEMENTED();
-    return gl::NoError();
+    return mRenderer->finish(context);
 }
 
 gl::Error ContextVk::initPipeline(const gl::Context *context)
@@ -295,7 +299,10 @@ gl::Error ContextVk::initPipeline(const gl::Context *context)
     return gl::NoError();
 }
 
-gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType drawType)
+gl::Error ContextVk::setupDraw(const gl::Context *context,
+                               GLenum mode,
+                               DrawType drawType,
+                               vk::CommandBuffer **commandBuffer)
 {
     if (mode != mCurrentDrawMode)
     {
@@ -325,23 +332,65 @@ gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType
     angle::MemoryBuffer *zeroBuf = nullptr;
     ANGLE_TRY(context->getZeroFilledBuffer(maxAttrib * sizeof(VkDeviceSize), &zeroBuf));
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(mRenderer->getStartedCommandBuffer(&commandBuffer));
-    ANGLE_TRY(mRenderer->ensureInRenderPass(context, vkFBO));
+    // TODO(jmadill): Need to link up the TextureVk to the Secondary CB.
+    vk::CommandBufferNode *renderNode = nullptr;
+    ANGLE_TRY(vkFBO->getRenderNode(context, &renderNode));
+
+    if (!renderNode->getInsideRenderPassCommands()->valid())
+    {
+        mVertexArrayDirty = true;
+        mTexturesDirty    = true;
+        ANGLE_TRY(renderNode->startRenderPassRecording(mRenderer, commandBuffer));
+    }
+    else
+    {
+        *commandBuffer = renderNode->getInsideRenderPassCommands();
+    }
+
+    // Ensure any writes to the VAO buffers are flushed before we read from them.
+    if (mVertexArrayDirty)
+    {
+        mVertexArrayDirty = false;
+        vkVAO->updateDrawDependencies(renderNode, programGL->getActiveAttribLocationsMask(),
+                                      queueSerial, drawType);
+    }
+
+    // Ensure any writes to the textures are flushed before we read from them.
+    if (mTexturesDirty)
+    {
+        mTexturesDirty = false;
+        // TODO(jmadill): Should probably merge this for loop with programVk's descriptor update.
+        const auto &completeTextures = state.getCompleteTextureCache();
+        for (const gl::SamplerBinding &samplerBinding : programGL->getSamplerBindings())
+        {
+            ASSERT(!samplerBinding.unreferenced);
+
+            // TODO(jmadill): Sampler arrays
+            ASSERT(samplerBinding.boundTextureUnits.size() == 1);
+
+            GLuint textureUnit         = samplerBinding.boundTextureUnits[0];
+            const gl::Texture *texture = completeTextures[textureUnit];
 
-    commandBuffer->bindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, mCurrentPipeline);
-    commandBuffer->bindVertexBuffers(0, maxAttrib, vertexHandles.data(),
-                                     reinterpret_cast<const VkDeviceSize *>(zeroBuf->data()));
+            // TODO(jmadill): Incomplete textures handling.
+            ASSERT(texture);
 
+            TextureVk *textureVk = vk::GetImpl(texture);
+            textureVk->updateDependencies(renderNode, mRenderer->getCurrentQueueSerial());
+        }
+    }
+
+    (*commandBuffer)->bindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, mCurrentPipeline);
+    (*commandBuffer)
+        ->bindVertexBuffers(0, maxAttrib, vertexHandles.data(),
+                            reinterpret_cast<const VkDeviceSize *>(zeroBuf->data()));
+
+    // Update the queue serial for the pipeline object.
     // TODO(jmadill): the queue serial should be bound to the pipeline.
-    setQueueSerial(queueSerial);
-    vkVAO->updateCurrentBufferSerials(programGL->getActiveAttribLocationsMask(), queueSerial,
-                                      drawType);
+    updateQueueSerial(queueSerial);
 
     // TODO(jmadill): Can probably use more dirty bits here.
-    ContextVk *contextVk = vk::GetImpl(context);
-    ANGLE_TRY(programVk->updateUniforms(contextVk));
-    programVk->updateTexturesDescriptorSet(contextVk);
+    ANGLE_TRY(programVk->updateUniforms(this));
+    programVk->updateTexturesDescriptorSet(this);
 
     // Bind the graphics descriptor sets.
     // TODO(jmadill): Handle multiple command buffers.
@@ -351,9 +400,9 @@ gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType
     if (!descriptorSets.empty() && ((setCount - firstSet) > 0))
     {
         const vk::PipelineLayout &pipelineLayout = programVk->getPipelineLayout();
-        commandBuffer->bindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, firstSet,
-                                          setCount - firstSet, &descriptorSets[firstSet], 0,
-                                          nullptr);
+        (*commandBuffer)
+            ->bindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, firstSet,
+                                 setCount - firstSet, &descriptorSets[firstSet], 0, nullptr);
     }
 
     return gl::NoError();
@@ -361,11 +410,8 @@ gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType
 
 gl::Error ContextVk::drawArrays(const gl::Context *context, GLenum mode, GLint first, GLsizei count)
 {
-    ANGLE_TRY(setupDraw(context, mode, DrawType::Arrays));
-
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(mRenderer->getStartedCommandBuffer(&commandBuffer));
-
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(setupDraw(context, mode, DrawType::Arrays, &commandBuffer));
     commandBuffer->draw(count, 1, first, 0);
     return gl::NoError();
 }
@@ -386,7 +432,8 @@ gl::Error ContextVk::drawElements(const gl::Context *context,
                                   GLenum type,
                                   const void *indices)
 {
-    ANGLE_TRY(setupDraw(context, mode, DrawType::Elements));
+    vk::CommandBuffer *commandBuffer;
+    ANGLE_TRY(setupDraw(context, mode, DrawType::Elements, &commandBuffer));
 
     if (indices)
     {
@@ -402,9 +449,6 @@ gl::Error ContextVk::drawElements(const gl::Context *context,
         return gl::InternalError() << "Unsigned byte translation is not yet implemented.";
     }
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(mRenderer->getStartedCommandBuffer(&commandBuffer));
-
     const gl::Buffer *elementArrayBuffer =
         mState.getState().getVertexArray()->getElementArrayBuffer().get();
     ASSERT(elementArrayBuffer);
@@ -444,18 +488,6 @@ VkDevice ContextVk::getDevice() const
     return mRenderer->getDevice();
 }
 
-vk::Error ContextVk::getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut)
-{
-    return mRenderer->getStartedCommandBuffer(commandBufferOut);
-}
-
-vk::Error ContextVk::submitCommands(vk::CommandBufferAndState *commandBuffer)
-{
-    setQueueSerial(mRenderer->getCurrentQueueSerial());
-    ANGLE_TRY(mRenderer->submitCommandBuffer(commandBuffer));
-    return vk::NoError();
-}
-
 gl::Error ContextVk::drawArraysIndirect(const gl::Context *context,
                                         GLenum mode,
                                         const void *indirect)
@@ -681,7 +713,7 @@ void ContextVk::syncState(const gl::Context *context, const gl::State::DirtyBits
                 WARN() << "DIRTY_BIT_RENDERBUFFER_BINDING unimplemented";
                 break;
             case gl::State::DIRTY_BIT_VERTEX_ARRAY_BINDING:
-                WARN() << "DIRTY_BIT_VERTEX_ARRAY_BINDING unimplemented";
+                mVertexArrayDirty = true;
                 break;
             case gl::State::DIRTY_BIT_DRAW_INDIRECT_BUFFER_BINDING:
                 WARN() << "DIRTY_BIT_DRAW_INDIRECT_BUFFER_BINDING unimplemented";
@@ -755,6 +787,7 @@ void ContextVk::syncState(const gl::Context *context, const gl::State::DirtyBits
     {
         ProgramVk *programVk = vk::GetImpl(glState.getProgram());
         programVk->invalidateTextures();
+        mTexturesDirty = true;
     }
 }
 
@@ -875,6 +908,13 @@ void ContextVk::invalidateCurrentPipeline()
     mRenderer->releaseResource(*this, &mCurrentPipeline);
 }
 
+void ContextVk::onVertexArrayChange()
+{
+    // TODO(jmadill): Does not handle dependent state changes.
+    mVertexArrayDirty = true;
+    invalidateCurrentPipeline();
+}
+
 gl::Error ContextVk::dispatchCompute(const gl::Context *context,
                                      GLuint numGroupsX,
                                      GLuint numGroupsY,

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -140,15 +140,6 @@ class ContextVk : public ContextImpl, public ResourceVk
     // Path object creation
     std::vector<PathImpl *> createPaths(GLsizei) override;
 
-    VkDevice getDevice() const;
-    vk::Error getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut);
-    vk::Error submitCommands(vk::CommandBufferAndState *commandBuffer);
-
-    RendererVk *getRenderer() { return mRenderer; }
-
-    // TODO(jmadill): Use pipeline cache.
-    void invalidateCurrentPipeline();
-
     gl::Error dispatchCompute(const gl::Context *context,
                               GLuint numGroupsX,
                               GLuint numGroupsY,
@@ -158,11 +149,22 @@ class ContextVk : public ContextImpl, public ResourceVk
     gl::Error memoryBarrier(const gl::Context *context, GLbitfield barriers) override;
     gl::Error memoryBarrierByRegion(const gl::Context *context, GLbitfield barriers) override;
 
+    VkDevice getDevice() const;
+    RendererVk *getRenderer() { return mRenderer; }
+
+    // TODO(jmadill): Use pipeline cache.
+    void invalidateCurrentPipeline();
+
+    void onVertexArrayChange();
+
     vk::DescriptorPool *getDescriptorPool();
 
   private:
     gl::Error initPipeline(const gl::Context *context);
-    gl::Error setupDraw(const gl::Context *context, GLenum mode, DrawType drawType);
+    gl::Error setupDraw(const gl::Context *context,
+                        GLenum mode,
+                        DrawType drawType,
+                        vk::CommandBuffer **commandBuffer);
 
     RendererVk *mRenderer;
     vk::Pipeline mCurrentPipeline;
@@ -187,6 +189,10 @@ class ContextVk : public ContextImpl, public ResourceVk
     // The descriptor pool is externally sychronized, so cannot be accessed from different threads
     // simulataneously. Hence, we keep it in the ContextVk instead of the RendererVk.
     vk::DescriptorPool mDescriptorPool;
+
+    // Triggers adding dependencies to the command graph.
+    bool mVertexArrayDirty;
+    bool mTexturesDirty;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/FramebufferVk.cpp

@@ -18,6 +18,7 @@
 #include "libANGLE/Display.h"
 #include "libANGLE/formatutils.h"
 #include "libANGLE/renderer/renderer_utils.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
 #include "libANGLE/renderer/vulkan/DisplayVk.h"
 #include "libANGLE/renderer/vulkan/RenderTargetVk.h"
@@ -56,12 +57,20 @@ FramebufferVk *FramebufferVk::CreateDefaultFBO(const gl::FramebufferState &state
 }
 
 FramebufferVk::FramebufferVk(const gl::FramebufferState &state)
-    : FramebufferImpl(state), mBackbuffer(nullptr), mRenderPassDesc(), mFramebuffer()
+    : FramebufferImpl(state),
+      mBackbuffer(nullptr),
+      mRenderPassDesc(),
+      mFramebuffer(),
+      mRenderNodeDirty(false)
 {
 }
 
 FramebufferVk::FramebufferVk(const gl::FramebufferState &state, WindowSurfaceVk *backbuffer)
-    : FramebufferImpl(state), mBackbuffer(backbuffer), mRenderPassDesc(), mFramebuffer()
+    : FramebufferImpl(state),
+      mBackbuffer(backbuffer),
+      mRenderPassDesc(),
+      mFramebuffer(),
+      mRenderNodeDirty(false)
 {
 }
 
@@ -110,8 +119,6 @@ gl::Error FramebufferVk::invalidateSub(const gl::Context *context,
 
 gl::Error FramebufferVk::clear(const gl::Context *context, GLbitfield mask)
 {
-    ContextVk *contextVk = vk::GetImpl(context);
-
     if (mState.getDepthAttachment() && (mask & GL_DEPTH_BUFFER_BIT) != 0)
     {
         // TODO(jmadill): Depth clear
@@ -143,8 +150,10 @@ gl::Error FramebufferVk::clear(const gl::Context *context, GLbitfield mask)
     const auto &size = attachment->getSize();
     const gl::Rectangle renderArea(0, 0, size.width, size.height);
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(contextVk->getStartedCommandBuffer(&commandBuffer));
+    RendererVk *renderer = vk::GetImpl(context)->getRenderer();
+
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
     for (const auto &colorAttachment : mState.getColorAttachments())
     {
@@ -249,11 +258,8 @@ gl::Error FramebufferVk::readPixels(const gl::Context *context,
                                            renderTarget->extents, vk::StagingUsage::Read,
                                            &stagingImage));
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(contextVk->getStartedCommandBuffer(&commandBuffer));
-
-    // End render pass if we're in one.
-    renderer->endRenderPass();
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
     stagingImage.getImage().changeLayoutTop(VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_GENERAL,
                                             commandBuffer);
@@ -283,7 +289,9 @@ gl::Error FramebufferVk::readPixels(const gl::Context *context,
 
     commandBuffer->copyImage(*readImage, stagingImage.getImage(), 1, &region);
 
-    ANGLE_TRY(renderer->submitAndFinishCommandBuffer(commandBuffer));
+    // Triggers a full finish.
+    // TODO(jmadill): Don't block on asynchronous readback.
+    ANGLE_TRY(renderer->finish(context));
 
     // TODO(jmadill): parameters
     uint8_t *mapPointer = nullptr;
@@ -338,7 +346,9 @@ void FramebufferVk::syncState(const gl::Context *context,
     // TODO(jmadill): Smarter update.
     mRenderPassDesc.reset();
     renderer->releaseResource(*this, &mFramebuffer);
-    renderer->onReleaseRenderPass(this);
+
+    // Trigger a new set of secondary commands next time we render to this FBO,.
+    mRenderNodeDirty = true;
 
     // TODO(jmadill): Use pipeline cache.
     contextVk->invalidateCurrentPipeline();
@@ -411,7 +421,7 @@ gl::ErrorOrResult<vk::Framebuffer *> FramebufferVk::getFramebuffer(const gl::Con
         if (colorAttachment.isAttached())
         {
             RenderTargetVk *renderTarget = nullptr;
-            ANGLE_TRY(colorAttachment.getRenderTarget<RenderTargetVk>(context, &renderTarget));
+            ANGLE_TRY(colorAttachment.getRenderTarget(context, &renderTarget));
             attachments.push_back(renderTarget->imageView->getHandle());
 
             ASSERT(attachmentsSize.empty() || attachmentsSize == colorAttachment.getSize());
@@ -423,7 +433,7 @@ gl::ErrorOrResult<vk::Framebuffer *> FramebufferVk::getFramebuffer(const gl::Con
     if (depthStencilAttachment && depthStencilAttachment->isAttached())
     {
         RenderTargetVk *renderTarget = nullptr;
-        ANGLE_TRY(depthStencilAttachment->getRenderTarget<RenderTargetVk>(context, &renderTarget));
+        ANGLE_TRY(depthStencilAttachment->getRenderTarget(context, &renderTarget));
         attachments.push_back(renderTarget->imageView->getHandle());
 
         ASSERT(attachmentsSize.empty() || attachmentsSize == depthStencilAttachment->getSize());
@@ -455,52 +465,28 @@ gl::Error FramebufferVk::getSamplePosition(size_t index, GLfloat *xy) const
     return gl::InternalError() << "getSamplePosition is unimplemented.";
 }
 
-gl::Error FramebufferVk::beginRenderPass(const gl::Context *context,
-                                         RendererVk *rendererVk,
-                                         vk::CommandBuffer *commandBuffer,
-                                         Serial queueSerial)
+gl::Error FramebufferVk::getRenderNode(const gl::Context *context, vk::CommandBufferNode **nodeOut)
 {
-    uint32_t attachmentIndex = 0;
-    vk::AttachmentOpsArray attachmentOps;
+    ContextVk *contextVk = vk::GetImpl(context);
+    RendererVk *renderer = contextVk->getRenderer();
+    Serial currentSerial = renderer->getCurrentQueueSerial();
 
-    // TODO(jmadill): Cache render targets.
-    for (const auto &colorAttachment : mState.getColorAttachments())
+    if (isCurrentlyRecording(currentSerial) && !mRenderNodeDirty)
     {
-        if (colorAttachment.isAttached())
-        {
-            RenderTargetVk *renderTarget = nullptr;
-            ANGLE_TRY(colorAttachment.getRenderTarget<RenderTargetVk>(context, &renderTarget));
-            renderTarget->resource->setQueueSerial(queueSerial);
-
-            // Fill in some default load and store ops.
-            attachmentOps.initDummyOp(attachmentIndex++, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
-        }
+        *nodeOut = getCurrentWriteNode(currentSerial);
+        ASSERT((*nodeOut)->getInsideRenderPassCommands()->valid());
+        return gl::NoError();
     }
 
-    const auto *depthStencilAttachment = mState.getDepthStencilAttachment();
-    if (depthStencilAttachment && depthStencilAttachment->isAttached())
-    {
-        RenderTargetVk *renderTarget = nullptr;
-        ANGLE_TRY(depthStencilAttachment->getRenderTarget<RenderTargetVk>(context, &renderTarget));
-        renderTarget->resource->setQueueSerial(queueSerial);
-
-        // Fill in some default load and store ops.
-        attachmentOps.initDummyOp(attachmentIndex++,
-                                  VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
-    }
+    vk::CommandBufferNode *node = getNewWriteNode(renderer);
 
     vk::Framebuffer *framebuffer = nullptr;
-    ANGLE_TRY_RESULT(getFramebuffer(context, rendererVk), framebuffer);
-    ASSERT(framebuffer && framebuffer->valid());
+    ANGLE_TRY_RESULT(getFramebuffer(context, renderer), framebuffer);
 
-    const vk::RenderPassDesc &desc = getRenderPassDesc(context);
-
-    vk::RenderPass *renderPass = nullptr;
-    ANGLE_TRY(rendererVk->getRenderPassWithOps(desc, attachmentOps, &renderPass));
-    ASSERT(renderPass && renderPass->valid());
+    const gl::State &glState = context->getGLState();
 
+    // Hard-code RenderPass to clear the first render target to the current clear value.
     // TODO(jmadill): Proper clear value implementation.
-    const gl::State &glState = context->getGLState();
     VkClearColorValue colorClear;
     memset(&colorClear, 0, sizeof(VkClearColorValue));
     colorClear.float32[0] = glState.getColorClearValue().red;
@@ -511,22 +497,39 @@ gl::Error FramebufferVk::beginRenderPass(const gl::Context *context,
     std::vector<VkClearValue> attachmentClearValues;
     attachmentClearValues.push_back({colorClear});
 
-    // Updated the cached image layout of the attachments in this FBO.
-    // For a default FBO, we need to call through to the WindowSurfaceVk
-    // TODO(jmadill): Iterate over all attachments.
-    RenderTargetVk *renderTarget = nullptr;
-    ANGLE_TRY(mState.getFirstColorAttachment()->getRenderTarget(context, &renderTarget));
-    renderTarget->image->updateLayout(VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+    node->storeRenderPassInfo(*framebuffer, glState.getViewport(), attachmentClearValues);
+
+    // Initialize RenderPass info.
+    // TODO(jmadill): Could cache this info, would require dependent state change messaging.
+    const auto &colorAttachments = mState.getColorAttachments();
+    for (size_t attachmentIndex = 0; attachmentIndex < colorAttachments.size(); ++attachmentIndex)
+    {
+        const auto &colorAttachment = colorAttachments[attachmentIndex];
+        if (colorAttachment.isAttached())
+        {
+            RenderTargetVk *renderTarget = nullptr;
+            ANGLE_SWALLOW_ERR(colorAttachment.getRenderTarget(context, &renderTarget));
 
-    commandBuffer->beginRenderPass(*renderPass, *framebuffer, glState.getViewport(), 1,
-                                   attachmentClearValues.data());
+            // TODO(jmadill): May need layout transition.
+            renderTarget->image->updateLayout(VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+            node->appendColorRenderTarget(currentSerial, renderTarget);
+        }
+    }
 
-    setQueueSerial(queueSerial);
-    if (mBackbuffer)
+    const gl::FramebufferAttachment *depthStencilAttachment = mState.getDepthOrStencilAttachment();
+    if (depthStencilAttachment && depthStencilAttachment->isAttached())
     {
-        mBackbuffer->setQueueSerial(queueSerial);
+        RenderTargetVk *renderTarget = nullptr;
+        ANGLE_SWALLOW_ERR(depthStencilAttachment->getRenderTarget(context, &renderTarget));
+
+        // TODO(jmadill): May need layout transition.
+        renderTarget->image->updateLayout(VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);
+        node->appendDepthStencilRenderTarget(currentSerial, renderTarget);
     }
 
+    mRenderNodeDirty = false;
+
+    *nodeOut = node;
     return gl::NoError();
 }
 

src/libANGLE/renderer/vulkan/FramebufferVk.h

@@ -84,12 +84,8 @@ class FramebufferVk : public FramebufferImpl, public ResourceVk
 
     gl::Error getSamplePosition(size_t index, GLfloat *xy) const override;
 
-    gl::Error beginRenderPass(const gl::Context *context,
-                              RendererVk *rendererVk,
-                              vk::CommandBuffer *commandBuffer,
-                              Serial queueSerial);
-
     const vk::RenderPassDesc &getRenderPassDesc(const gl::Context *context);
+    gl::Error getRenderNode(const gl::Context *context, vk::CommandBufferNode **nodeOut);
 
   private:
     FramebufferVk(const gl::FramebufferState &state);
@@ -102,6 +98,7 @@ class FramebufferVk : public FramebufferImpl, public ResourceVk
 
     Optional<vk::RenderPassDesc> mRenderPassDesc;
     vk::Framebuffer mFramebuffer;
+    bool mRenderNodeDirty;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -17,6 +17,7 @@
 #include "common/debug.h"
 #include "common/system_utils.h"
 #include "libANGLE/renderer/driver_utils.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/CompilerVk.h"
 #include "libANGLE/renderer/vulkan/FramebufferVk.h"
 #include "libANGLE/renderer/vulkan/GlslangWrapper.h"
@@ -179,6 +180,28 @@ vk::Error RenderPassCache::getRenderPassWithOps(VkDevice device,
     return vk::NoError();
 }
 
+// CommandBatch implementation.
+RendererVk::CommandBatch::CommandBatch()
+{
+}
+
+RendererVk::CommandBatch::~CommandBatch()
+{
+}
+
+RendererVk::CommandBatch::CommandBatch(CommandBatch &&other)
+    : commandPool(std::move(other.commandPool)), fence(std::move(other.fence)), serial(other.serial)
+{
+}
+
+RendererVk::CommandBatch &RendererVk::CommandBatch::operator=(CommandBatch &&other)
+{
+    std::swap(commandPool, other.commandPool);
+    std::swap(fence, other.fence);
+    std::swap(serial, other.serial);
+    return *this;
+}
+
 // RendererVk implementation.
 RendererVk::RendererVk()
     : mCapsInitialized(false),
@@ -192,16 +215,16 @@ RendererVk::RendererVk()
       mGlslangWrapper(nullptr),
       mLastCompletedQueueSerial(mQueueSerialFactory.generate()),
       mCurrentQueueSerial(mQueueSerialFactory.generate()),
-      mInFlightCommands(),
-      mCurrentRenderPassFramebuffer(nullptr)
+      mInFlightCommands()
 {
 }
 
 RendererVk::~RendererVk()
 {
-    if (!mInFlightCommands.empty() || !mInFlightFences.empty() || !mGarbage.empty())
+    if (!mInFlightCommands.empty() || !mGarbage.empty())
     {
-        vk::Error error = finish();
+        // TODO(jmadill): Not nice to pass nullptr here, but shouldn't be a problem.
+        vk::Error error = finish(nullptr);
         if (error.isError())
         {
             ERR() << "Error during VK shutdown: " << error;
@@ -216,11 +239,6 @@ RendererVk::~RendererVk()
         mGlslangWrapper = nullptr;
     }
 
-    if (mCommandBuffer.valid())
-    {
-        mCommandBuffer.destroy(mDevice, mCommandPool);
-    }
-
     if (mCommandPool.valid())
     {
         mCommandPool.destroy(mDevice);
@@ -519,10 +537,9 @@ vk::Error RendererVk::initializeDevice(uint32_t queueFamilyIndex)
 
     // Initialize the command pool now that we know the queue family index.
     VkCommandPoolCreateInfo commandPoolInfo;
-    commandPoolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
-    commandPoolInfo.pNext = nullptr;
-    // TODO(jmadill): Investigate transient command buffers.
-    commandPoolInfo.flags            = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
+    commandPoolInfo.sType            = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+    commandPoolInfo.pNext            = nullptr;
+    commandPoolInfo.flags            = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT;
     commandPoolInfo.queueFamilyIndex = mCurrentQueueFamilyIndex;
 
     ANGLE_TRY(mCommandPool.init(mDevice, commandPoolInfo));
@@ -665,74 +682,32 @@ const gl::Limitations &RendererVk::getNativeLimitations() const
     return mNativeLimitations;
 }
 
-vk::Error RendererVk::getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut)
+const vk::CommandPool &RendererVk::getCommandPool() const
 {
-    ANGLE_TRY(mCommandBuffer.ensureStarted(mDevice, mCommandPool, VK_COMMAND_BUFFER_LEVEL_PRIMARY));
-    *commandBufferOut = &mCommandBuffer;
-    return vk::NoError();
+    return mCommandPool;
 }
 
-vk::Error RendererVk::submitCommandBuffer(vk::CommandBufferAndState *commandBuffer)
+vk::Error RendererVk::finish(const gl::Context *context)
 {
-    ANGLE_TRY(commandBuffer->ensureFinished());
-
-    VkFenceCreateInfo fenceInfo;
-    fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-    fenceInfo.pNext = nullptr;
-    fenceInfo.flags = 0;
-
-    VkSubmitInfo submitInfo;
-    submitInfo.sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-    submitInfo.pNext                = nullptr;
-    submitInfo.waitSemaphoreCount   = 0;
-    submitInfo.pWaitSemaphores      = nullptr;
-    submitInfo.pWaitDstStageMask    = nullptr;
-    submitInfo.commandBufferCount   = 1;
-    submitInfo.pCommandBuffers      = commandBuffer->ptr();
-    submitInfo.signalSemaphoreCount = 0;
-    submitInfo.pSignalSemaphores    = nullptr;
-
-    // TODO(jmadill): Investigate how to properly submit command buffers.
-    ANGLE_TRY(submit(submitInfo));
-
-    return vk::NoError();
-}
-
-vk::Error RendererVk::submitAndFinishCommandBuffer(vk::CommandBufferAndState *commandBuffer)
-{
-    ANGLE_TRY(submitCommandBuffer(commandBuffer));
-    ANGLE_TRY(finish());
-
-    return vk::NoError();
-}
-
-vk::Error RendererVk::submitCommandsWithSync(vk::CommandBufferAndState *commandBuffer,
-                                             const vk::Semaphore &waitSemaphore,
-                                             const vk::Semaphore &signalSemaphore)
-{
-    ANGLE_TRY(commandBuffer->end());
-
-    VkPipelineStageFlags waitStageMask  = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
-
-    VkSubmitInfo submitInfo;
-    submitInfo.sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-    submitInfo.pNext                = nullptr;
-    submitInfo.waitSemaphoreCount   = 1;
-    submitInfo.pWaitSemaphores      = waitSemaphore.ptr();
-    submitInfo.pWaitDstStageMask    = &waitStageMask;
-    submitInfo.commandBufferCount   = 1;
-    submitInfo.pCommandBuffers      = commandBuffer->ptr();
-    submitInfo.signalSemaphoreCount = 1;
-    submitInfo.pSignalSemaphores    = signalSemaphore.ptr();
+    if (!mOpenCommandGraph.empty())
+    {
+        vk::CommandBuffer commandBatch;
+        ANGLE_TRY(flushCommandGraph(context, &commandBatch));
 
-    // TODO(jmadill): Investigate how to properly queue command buffer work.
-    ANGLE_TRY(submitFrame(submitInfo));
+        VkSubmitInfo submitInfo;
+        submitInfo.sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+        submitInfo.pNext                = nullptr;
+        submitInfo.waitSemaphoreCount   = 0;
+        submitInfo.pWaitSemaphores      = nullptr;
+        submitInfo.pWaitDstStageMask    = nullptr;
+        submitInfo.commandBufferCount   = 1;
+        submitInfo.pCommandBuffers      = commandBatch.ptr();
+        submitInfo.signalSemaphoreCount = 0;
+        submitInfo.pSignalSemaphores    = nullptr;
 
-    return vk::NoError();
-}
+        ANGLE_TRY(submitFrame(submitInfo, std::move(commandBatch)));
+    }
 
-vk::Error RendererVk::finish()
-{
     ASSERT(mQueue != VK_NULL_HANDLE);
     ANGLE_VK_TRY(vkQueueWaitIdle(mQueue));
     freeAllInFlightResources();
@@ -741,15 +716,10 @@ vk::Error RendererVk::finish()
 
 void RendererVk::freeAllInFlightResources()
 {
-    for (auto &fence : mInFlightFences)
-    {
-        fence.get().destroy(mDevice);
-    }
-    mInFlightFences.clear();
-
-    for (auto &command : mInFlightCommands)
+    for (CommandBatch &batch : mInFlightCommands)
     {
-        command.get().destroy(mDevice, mCommandPool);
+        batch.fence.destroy(mDevice);
+        batch.commandPool.destroy(mDevice);
     }
     mInFlightCommands.clear();
 
@@ -762,51 +732,29 @@ void RendererVk::freeAllInFlightResources()
 
 vk::Error RendererVk::checkInFlightCommands()
 {
-    size_t finishedIndex = 0;
+    int finishedCount = 0;
 
-    // Check if any in-flight command buffers are finished.
-    for (size_t index = 0; index < mInFlightFences.size(); index++)
+    for (CommandBatch &batch : mInFlightCommands)
     {
-        auto *inFlightFence = &mInFlightFences[index];
-
-        VkResult result = inFlightFence->get().getStatus(mDevice);
+        VkResult result = batch.fence.getStatus(mDevice);
         if (result == VK_NOT_READY)
             break;
-        ANGLE_VK_TRY(result);
-        finishedIndex = index + 1;
 
-        // Release the fence handle.
-        // TODO(jmadill): Re-use fences.
-        inFlightFence->get().destroy(mDevice);
-    }
-
-    if (finishedIndex == 0)
-        return vk::NoError();
-
-    Serial finishedSerial = mInFlightFences[finishedIndex - 1].queueSerial();
-    mInFlightFences.erase(mInFlightFences.begin(), mInFlightFences.begin() + finishedIndex);
-
-    size_t completedCBIndex = 0;
-    for (size_t cbIndex = 0; cbIndex < mInFlightCommands.size(); ++cbIndex)
-    {
-        auto *inFlightCB = &mInFlightCommands[cbIndex];
-        if (inFlightCB->queueSerial() > finishedSerial)
-            break;
+        ANGLE_VK_TRY(result);
+        ASSERT(batch.serial > mLastCompletedQueueSerial);
+        mLastCompletedQueueSerial = batch.serial;
 
-        completedCBIndex = cbIndex + 1;
-        inFlightCB->get().destroy(mDevice, mCommandPool);
+        batch.fence.destroy(mDevice);
+        batch.commandPool.destroy(mDevice);
+        ++finishedCount;
     }
 
-    if (completedCBIndex == 0)
-        return vk::NoError();
-
-    mInFlightCommands.erase(mInFlightCommands.begin(),
-                            mInFlightCommands.begin() + completedCBIndex);
+    mInFlightCommands.erase(mInFlightCommands.begin(), mInFlightCommands.begin() + finishedCount);
 
     size_t freeIndex = 0;
     for (; freeIndex < mGarbage.size(); ++freeIndex)
     {
-        if (!mGarbage[freeIndex].destroyIfComplete(mDevice, finishedSerial))
+        if (!mGarbage[freeIndex].destroyIfComplete(mDevice, mLastCompletedQueueSerial))
             break;
     }
 
@@ -819,38 +767,23 @@ vk::Error RendererVk::checkInFlightCommands()
     return vk::NoError();
 }
 
-vk::Error RendererVk::submit(const VkSubmitInfo &submitInfo)
+vk::Error RendererVk::submitFrame(const VkSubmitInfo &submitInfo, vk::CommandBuffer &&commandBuffer)
 {
-    ANGLE_VK_TRY(vkQueueSubmit(mQueue, 1, &submitInfo, VK_NULL_HANDLE));
-
-    // Store this command buffer in the in-flight list.
-    mInFlightCommands.emplace_back(std::move(mCommandBuffer), mCurrentQueueSerial);
-
-    // Sanity check.
-    ASSERT(mInFlightCommands.size() < 1000u);
-
-    // Increment the queue serial. If this fails, we should restart ANGLE.
-    // TODO(jmadill): Overflow check.
-    mCurrentQueueSerial = mQueueSerialFactory.generate();
-
-    return vk::NoError();
-}
-
-vk::Error RendererVk::submitFrame(const VkSubmitInfo &submitInfo)
-{
-    VkFenceCreateInfo createInfo;
-    createInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-    createInfo.pNext = nullptr;
-    createInfo.flags = 0;
+    VkFenceCreateInfo fenceInfo;
+    fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+    fenceInfo.pNext = nullptr;
+    fenceInfo.flags = 0;
 
-    vk::Fence fence;
-    ANGLE_TRY(fence.init(mDevice, createInfo));
+    CommandBatch batch;
+    ANGLE_TRY(batch.fence.init(mDevice, fenceInfo));
 
-    ANGLE_VK_TRY(vkQueueSubmit(mQueue, 1, &submitInfo, fence.getHandle()));
+    ANGLE_VK_TRY(vkQueueSubmit(mQueue, 1, &submitInfo, batch.fence.getHandle()));
 
     // Store this command buffer in the in-flight list.
-    mInFlightFences.emplace_back(std::move(fence), mCurrentQueueSerial);
-    mInFlightCommands.emplace_back(std::move(mCommandBuffer), mCurrentQueueSerial);
+    batch.commandPool = std::move(mCommandPool);
+    batch.serial      = mCurrentQueueSerial;
+
+    mInFlightCommands.emplace_back(std::move(batch));
 
     // Sanity check.
     ASSERT(mInFlightCommands.size() < 1000u);
@@ -861,6 +794,19 @@ vk::Error RendererVk::submitFrame(const VkSubmitInfo &submitInfo)
 
     ANGLE_TRY(checkInFlightCommands());
 
+    // Simply null out the command buffer here - it was allocated using the command pool.
+    commandBuffer.releaseHandle();
+
+    // Reallocate the command pool for next frame.
+    // TODO(jmadill): Consider reusing command pools.
+    VkCommandPoolCreateInfo poolInfo;
+    poolInfo.sType            = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+    poolInfo.pNext            = nullptr;
+    poolInfo.flags            = 0;
+    poolInfo.queueFamilyIndex = mCurrentQueueFamilyIndex;
+
+    mCommandPool.init(mDevice, poolInfo);
+
     return vk::NoError();
 }
 
@@ -885,40 +831,6 @@ Serial RendererVk::getCurrentQueueSerial() const
     return mCurrentQueueSerial;
 }
 
-gl::Error RendererVk::ensureInRenderPass(const gl::Context *context, FramebufferVk *framebufferVk)
-{
-    if (mCurrentRenderPassFramebuffer == framebufferVk)
-    {
-        return gl::NoError();
-    }
-
-    if (mCurrentRenderPassFramebuffer)
-    {
-        endRenderPass();
-    }
-    ANGLE_TRY(framebufferVk->beginRenderPass(context, this, &mCommandBuffer, mCurrentQueueSerial));
-    mCurrentRenderPassFramebuffer = framebufferVk;
-    return gl::NoError();
-}
-
-void RendererVk::endRenderPass()
-{
-    if (mCurrentRenderPassFramebuffer)
-    {
-        ASSERT(mCommandBuffer.started());
-        mCommandBuffer.endRenderPass();
-        mCurrentRenderPassFramebuffer = nullptr;
-    }
-}
-
-void RendererVk::onReleaseRenderPass(const FramebufferVk *framebufferVk)
-{
-    if (mCurrentRenderPassFramebuffer == framebufferVk)
-    {
-        endRenderPass();
-    }
-}
-
 bool RendererVk::isResourceInUse(const ResourceVk &resource)
 {
     return isSerialInUse(resource.getQueueSerial());
@@ -944,4 +856,109 @@ vk::Error RendererVk::getRenderPassWithOps(const vk::RenderPassDesc &desc,
                                                  renderPassOut);
 }
 
+vk::CommandBufferNode *RendererVk::allocateCommandNode()
+{
+    // TODO(jmadill): Use a pool allocator for the CPU node allocations.
+    vk::CommandBufferNode *newCommands = new vk::CommandBufferNode();
+    mOpenCommandGraph.emplace_back(newCommands);
+    return newCommands;
+}
+
+vk::Error RendererVk::flushCommandGraph(const gl::Context *context, vk::CommandBuffer *commandBatch)
+{
+    VkCommandBufferAllocateInfo primaryInfo;
+    primaryInfo.sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+    primaryInfo.pNext              = nullptr;
+    primaryInfo.commandPool        = mCommandPool.getHandle();
+    primaryInfo.level              = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+    primaryInfo.commandBufferCount = 1;
+
+    ANGLE_TRY(commandBatch->init(mDevice, primaryInfo));
+
+    if (mOpenCommandGraph.empty())
+    {
+        return vk::NoError();
+    }
+
+    std::vector<vk::CommandBufferNode *> nodeStack;
+
+    VkCommandBufferBeginInfo beginInfo;
+    beginInfo.sType            = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+    beginInfo.pNext            = nullptr;
+    beginInfo.flags            = 0;
+    beginInfo.pInheritanceInfo = nullptr;
+
+    ANGLE_TRY(commandBatch->begin(beginInfo));
+
+    for (vk::CommandBufferNode *topLevelNode : mOpenCommandGraph)
+    {
+        // Only process commands that don't have child commands. The others will be pulled in
+        // automatically. Also skip commands that have already been visited.
+        if (topLevelNode->isDependency() ||
+            topLevelNode->visitedState() != vk::VisitedState::Unvisited)
+            continue;
+
+        nodeStack.push_back(topLevelNode);
+
+        while (!nodeStack.empty())
+        {
+            vk::CommandBufferNode *node = nodeStack.back();
+
+            switch (node->visitedState())
+            {
+                case vk::VisitedState::Unvisited:
+                    node->visitDependencies(&nodeStack);
+                    break;
+                case vk::VisitedState::Ready:
+                    ANGLE_TRY(node->visitAndExecute(this, commandBatch));
+                    nodeStack.pop_back();
+                    break;
+                case vk::VisitedState::Visited:
+                    nodeStack.pop_back();
+                    break;
+                default:
+                    UNREACHABLE();
+                    break;
+            }
+        }
+    }
+
+    ANGLE_TRY(commandBatch->end());
+    return vk::NoError();
+}
+
+void RendererVk::resetCommandGraph()
+{
+    // TODO(jmadill): Use pool allocation so we don't need to deallocate command graph.
+    for (vk::CommandBufferNode *node : mOpenCommandGraph)
+    {
+        delete node;
+    }
+    mOpenCommandGraph.clear();
+}
+
+vk::Error RendererVk::flush(const gl::Context *context,
+                            const vk::Semaphore &waitSemaphore,
+                            const vk::Semaphore &signalSemaphore)
+{
+    vk::CommandBuffer commandBatch;
+    ANGLE_TRY(flushCommandGraph(context, &commandBatch));
+
+    VkPipelineStageFlags waitStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
+
+    VkSubmitInfo submitInfo;
+    submitInfo.sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+    submitInfo.pNext                = nullptr;
+    submitInfo.waitSemaphoreCount   = 1;
+    submitInfo.pWaitSemaphores      = waitSemaphore.ptr();
+    submitInfo.pWaitDstStageMask    = &waitStageMask;
+    submitInfo.commandBufferCount   = 1;
+    submitInfo.pCommandBuffers      = commandBatch.ptr();
+    submitInfo.signalSemaphoreCount = 1;
+    submitInfo.pSignalSemaphores    = signalSemaphore.ptr();
+
+    ANGLE_TRY(submitFrame(submitInfo, std::move(commandBatch)));
+    return vk::NoError();
+}
+
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -79,14 +79,12 @@ class RendererVk : angle::NonCopyable
 
     vk::ErrorOrResult<uint32_t> selectPresentQueueForSurface(VkSurfaceKHR surface);
 
-    // TODO(jmadill): Use ContextImpl for command buffers to enable threaded contexts.
-    vk::Error getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut);
-    vk::Error submitCommandBuffer(vk::CommandBufferAndState *commandBuffer);
-    vk::Error submitAndFinishCommandBuffer(vk::CommandBufferAndState *commandBuffer);
-    vk::Error submitCommandsWithSync(vk::CommandBufferAndState *commandBuffer,
-                                     const vk::Semaphore &waitSemaphore,
-                                     const vk::Semaphore &signalSemaphore);
-    vk::Error finish();
+    vk::Error finish(const gl::Context *context);
+    vk::Error flush(const gl::Context *context,
+                    const vk::Semaphore &waitSemaphore,
+                    const vk::Semaphore &signalSemaphore);
+
+    const vk::CommandPool &getCommandPool() const;
 
     const gl::Caps &getNativeCaps() const;
     const gl::TextureCapsMap &getNativeTextureCaps() const;
@@ -130,13 +128,6 @@ class RendererVk : angle::NonCopyable
 
     const vk::MemoryProperties &getMemoryProperties() const { return mMemoryProperties; }
 
-    // TODO(jmadill): Don't keep a single renderpass in the Renderer.
-    gl::Error ensureInRenderPass(const gl::Context *context, FramebufferVk *framebufferVk);
-    void endRenderPass();
-
-    // This is necessary to update the cached current RenderPass Framebuffer.
-    void onReleaseRenderPass(const FramebufferVk *framebufferVk);
-
     // TODO(jmadill): We could pass angle::Format::ID here.
     const vk::Format &getFormat(GLenum internalFormat) const
     {
@@ -149,16 +140,22 @@ class RendererVk : angle::NonCopyable
                                    const vk::AttachmentOpsArray &ops,
                                    vk::RenderPass **renderPassOut);
 
+    // This should only be called from ResourceVk.
+    // TODO(jmadill): Keep in ContextVk to enable threaded rendering.
+    vk::CommandBufferNode *allocateCommandNode();
+
   private:
+    vk::Error initializeDevice(uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
     void generateCaps(gl::Caps *outCaps,
                       gl::TextureCapsMap *outTextureCaps,
                       gl::Extensions *outExtensions,
                       gl::Limitations *outLimitations) const;
-    vk::Error submit(const VkSubmitInfo &submitInfo);
-    vk::Error submitFrame(const VkSubmitInfo &submitInfo);
+    vk::Error submitFrame(const VkSubmitInfo &submitInfo, vk::CommandBuffer &&commandBatch);
     vk::Error checkInFlightCommands();
     void freeAllInFlightResources();
+    vk::Error flushCommandGraph(const gl::Context *context, vk::CommandBuffer *commandBatch);
+    void resetCommandGraph();
 
     mutable bool mCapsInitialized;
     mutable gl::Caps mNativeCaps;
@@ -166,8 +163,6 @@ class RendererVk : angle::NonCopyable
     mutable gl::Extensions mNativeExtensions;
     mutable gl::Limitations mNativeLimitations;
 
-    vk::Error initializeDevice(uint32_t queueFamilyIndex);
-
     VkInstance mInstance;
     bool mEnableValidationLayers;
     VkDebugReportCallbackEXT mDebugReportCallback;
@@ -178,21 +173,30 @@ class RendererVk : angle::NonCopyable
     uint32_t mCurrentQueueFamilyIndex;
     VkDevice mDevice;
     vk::CommandPool mCommandPool;
-    vk::CommandBufferAndState mCommandBuffer;
     GlslangWrapper *mGlslangWrapper;
     SerialFactory mQueueSerialFactory;
     Serial mLastCompletedQueueSerial;
     Serial mCurrentQueueSerial;
-    std::vector<vk::CommandBufferAndSerial> mInFlightCommands;
-    std::vector<vk::FenceAndSerial> mInFlightFences;
+
+    struct CommandBatch final : angle::NonCopyable
+    {
+        CommandBatch();
+        ~CommandBatch();
+        CommandBatch(CommandBatch &&other);
+        CommandBatch &operator=(CommandBatch &&other);
+
+        vk::CommandPool commandPool;
+        vk::Fence fence;
+        Serial serial;
+    };
+
+    std::vector<CommandBatch> mInFlightCommands;
     std::vector<vk::GarbageObject> mGarbage;
     vk::MemoryProperties mMemoryProperties;
     vk::FormatTable mFormatTable;
 
-    // TODO(jmadill): Don't keep a single renderpass in the Renderer.
-    FramebufferVk *mCurrentRenderPassFramebuffer;
-
     RenderPassCache mRenderPassCache;
+    std::vector<vk::CommandBufferNode *> mOpenCommandGraph;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -192,7 +192,7 @@ void WindowSurfaceVk::destroy(const egl::Display *display)
     VkDevice device            = rendererVk->getDevice();
     VkInstance instance        = rendererVk->getInstance();
 
-    rendererVk->finish();
+    rendererVk->finish(display->getProxyContext());
 
     mAcquireNextImageSemaphore.destroy(device);
 
@@ -359,9 +359,9 @@ vk::Error WindowSurfaceVk::initializeImpl(RendererVk *renderer)
     std::vector<VkImage> swapchainImages(imageCount);
     ANGLE_VK_TRY(vkGetSwapchainImagesKHR(device, mSwapchain, &imageCount, swapchainImages.data()));
 
-    // CommandBuffer is a singleton in the Renderer.
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(renderer->getStartedCommandBuffer(&commandBuffer));
+    // Allocate a command buffer for clearing our images to black.
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
     VkClearColorValue transparentBlack;
     transparentBlack.float32[0] = 0.0f;
@@ -409,8 +409,6 @@ vk::Error WindowSurfaceVk::initializeImpl(RendererVk *renderer)
         ANGLE_TRY(member.commandsCompleteSemaphore.init(device));
     }
 
-    ANGLE_TRY(renderer->submitAndFinishCommandBuffer(commandBuffer));
-
     // Get the first available swapchain iamge.
     ANGLE_TRY(nextSwapchainImage(renderer));
 
@@ -427,20 +425,17 @@ egl::Error WindowSurfaceVk::swap(const gl::Context *context)
     const DisplayVk *displayVk = vk::GetImpl(context->getCurrentDisplay());
     RendererVk *renderer       = displayVk->getRenderer();
 
-    vk::CommandBufferAndState *currentCB = nullptr;
-    ANGLE_TRY(renderer->getStartedCommandBuffer(&currentCB));
-
-    // End render pass
-    renderer->endRenderPass();
+    vk::CommandBuffer *swapCommands = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &swapCommands));
 
     auto &image = mSwapchainImages[mCurrentSwapchainImageIndex];
 
     image.image.changeLayoutWithStages(VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
                                        VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
-                                       VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, currentCB);
+                                       VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, swapCommands);
 
-    ANGLE_TRY(renderer->submitCommandsWithSync(currentCB, image.imageAcquiredSemaphore,
-                                               image.commandsCompleteSemaphore));
+    ANGLE_TRY(
+        renderer->flush(context, image.imageAcquiredSemaphore, image.commandsCompleteSemaphore));
 
     VkPresentInfoKHR presentInfo;
     presentInfo.sType              = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;

src/libANGLE/renderer/vulkan/TextureVk.cpp

@@ -177,7 +177,6 @@ gl::Error TextureVk::setImage(const gl::Context *context,
     mRenderTarget.resource  = this;
 
     // Handle initial data.
-    // TODO(jmadill): Consider re-using staging texture.
     if (pixels)
     {
         ANGLE_TRY(setSubImageImpl(contextVk, formatInfo, unpack, type, pixels));
@@ -258,13 +257,8 @@ gl::Error TextureVk::setSubImageImpl(ContextVk *contextVk,
 
     stagingImage.getDeviceMemory().unmap(device);
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(contextVk->getStartedCommandBuffer(&commandBuffer));
-    setQueueSerial(renderer->getCurrentQueueSerial());
-
-    // Ensure we aren't in a render pass.
-    // TODO(jmadill): Command reordering.
-    renderer->endRenderPass();
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
     stagingImage.getImage().changeLayoutWithStages(
         VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
@@ -277,7 +271,8 @@ gl::Error TextureVk::setSubImageImpl(ContextVk *contextVk,
     commandBuffer->copySingleImage(stagingImage.getImage(), mImage, wholeRegion,
                                    VK_IMAGE_ASPECT_COLOR_BIT);
 
-    // TODO(jmadill): Re-use staging images.
+    // Immediately release staging image.
+    // TODO(jmadill): Staging image re-use.
     renderer->releaseObject(renderer->getCurrentQueueSerial(), &stagingImage);
     return gl::NoError();
 }

src/libANGLE/renderer/vulkan/VertexArrayVk.cpp

@@ -13,6 +13,7 @@
 
 #include "libANGLE/Context.h"
 #include "libANGLE/renderer/vulkan/BufferVk.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
 #include "libANGLE/renderer/vulkan/formatutilsvk.h"
 
@@ -48,7 +49,7 @@ void VertexArrayVk::syncState(const gl::Context *context,
     // Invalidate current pipeline.
     // TODO(jmadill): Use pipeline cache.
     ContextVk *contextVk = vk::GetImpl(context);
-    contextVk->invalidateCurrentPipeline();
+    contextVk->onVertexArrayChange();
 
     // Invalidate the vertex descriptions.
     invalidateVertexDescriptions();
@@ -107,22 +108,23 @@ const gl::AttribArray<VkBuffer> &VertexArrayVk::getCurrentArrayBufferHandles() c
     return mCurrentArrayBufferHandles;
 }
 
-void VertexArrayVk::updateCurrentBufferSerials(const gl::AttributesMask &activeAttribsMask,
-                                               Serial serial,
-                                               DrawType drawType)
+void VertexArrayVk::updateDrawDependencies(vk::CommandBufferNode *readNode,
+                                           const gl::AttributesMask &activeAttribsMask,
+                                           Serial serial,
+                                           DrawType drawType)
 {
     // Handle the bound array buffers.
     for (auto attribIndex : activeAttribsMask)
     {
         ASSERT(mCurrentArrayBufferResources[attribIndex]);
-        mCurrentArrayBufferResources[attribIndex]->setQueueSerial(serial);
+        mCurrentArrayBufferResources[attribIndex]->updateDependencies(readNode, serial);
     }
 
     // Handle the bound element array buffer.
     if (drawType == DrawType::Elements)
     {
         ASSERT(mCurrentElementArrayBufferResource);
-        mCurrentElementArrayBufferResource->setQueueSerial(serial);
+        mCurrentElementArrayBufferResource->updateDependencies(readNode, serial);
     }
 }
 

src/libANGLE/renderer/vulkan/VertexArrayVk.h

@@ -30,9 +30,10 @@ class VertexArrayVk : public VertexArrayImpl
 
     const gl::AttribArray<VkBuffer> &getCurrentArrayBufferHandles() const;
 
-    void updateCurrentBufferSerials(const gl::AttributesMask &activeAttribsMask,
-                                    Serial serial,
-                                    DrawType drawType);
+    void updateDrawDependencies(vk::CommandBufferNode *readNode,
+                                const gl::AttributesMask &activeAttribsMask,
+                                Serial serial,
+                                DrawType drawType);
 
     void invalidateVertexDescriptions();
     void updateVertexDescriptions(const gl::Context *context);

src/libANGLE/renderer/vulkan/renderervk_utils.cpp

@@ -9,8 +9,11 @@
 
 #include "renderervk_utils.h"
 
+#include "libANGLE/Context.h"
 #include "libANGLE/SizedMRUCache.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
+#include "libANGLE/renderer/vulkan/RenderTargetVk.h"
 #include "libANGLE/renderer/vulkan/RendererVk.h"
 
 namespace rx
@@ -315,6 +318,20 @@ CommandBuffer::CommandBuffer()
 {
 }
 
+VkCommandBuffer CommandBuffer::releaseHandle()
+{
+    VkCommandBuffer handle = mHandle;
+    mHandle                = nullptr;
+    return handle;
+}
+
+Error CommandBuffer::init(VkDevice device, const VkCommandBufferAllocateInfo &createInfo)
+{
+    ASSERT(!valid());
+    ANGLE_VK_TRY(vkAllocateCommandBuffers(device, &createInfo, &mHandle));
+    return NoError();
+}
+
 Error CommandBuffer::begin(const VkCommandBufferBeginInfo &info)
 {
     ASSERT(valid());
@@ -366,13 +383,6 @@ void CommandBuffer::destroy(VkDevice device, const vk::CommandPool &commandPool)
     }
 }
 
-Error CommandBuffer::init(VkDevice device, const VkCommandBufferAllocateInfo &createInfo)
-{
-    ASSERT(!valid());
-    ANGLE_VK_TRY(vkAllocateCommandBuffers(device, &createInfo, &mHandle));
-    return NoError();
-}
-
 void CommandBuffer::copyBuffer(const vk::Buffer &srcBuffer,
                                const vk::Buffer &destBuffer,
                                uint32_t regionCount,
@@ -440,27 +450,11 @@ void CommandBuffer::copyImage(const vk::Image &srcImage,
                    dstImage.getCurrentLayout(), 1, regions);
 }
 
-void CommandBuffer::beginRenderPass(const RenderPass &renderPass,
-                                    const Framebuffer &framebuffer,
-                                    const gl::Rectangle &renderArea,
-                                    uint32_t clearValueCount,
-                                    const VkClearValue *clearValues)
+void CommandBuffer::beginRenderPass(const VkRenderPassBeginInfo &beginInfo,
+                                    VkSubpassContents subpassContents)
 {
-    ASSERT(mHandle != VK_NULL_HANDLE);
-
-    VkRenderPassBeginInfo beginInfo;
-    beginInfo.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
-    beginInfo.pNext                    = nullptr;
-    beginInfo.renderPass               = renderPass.getHandle();
-    beginInfo.framebuffer              = framebuffer.getHandle();
-    beginInfo.renderArea.offset.x      = static_cast<uint32_t>(renderArea.x);
-    beginInfo.renderArea.offset.y      = static_cast<uint32_t>(renderArea.y);
-    beginInfo.renderArea.extent.width  = static_cast<uint32_t>(renderArea.width);
-    beginInfo.renderArea.extent.height = static_cast<uint32_t>(renderArea.height);
-    beginInfo.clearValueCount          = clearValueCount;
-    beginInfo.pClearValues             = clearValues;
-
-    vkCmdBeginRenderPass(mHandle, &beginInfo, VK_SUBPASS_CONTENTS_INLINE);
+    ASSERT(valid());
+    vkCmdBeginRenderPass(mHandle, &beginInfo, subpassContents);
 }
 
 void CommandBuffer::endRenderPass()
@@ -525,6 +519,13 @@ void CommandBuffer::bindDescriptorSets(VkPipelineBindPoint bindPoint,
                             descriptorSets, dynamicOffsetCount, dynamicOffsets);
 }
 
+void CommandBuffer::executeCommands(uint32_t commandBufferCount,
+                                    const vk::CommandBuffer *commandBuffers)
+{
+    ASSERT(valid());
+    vkCmdExecuteCommands(mHandle, commandBufferCount, commandBuffers[0].ptr());
+}
+
 // Image implementation.
 Image::Image() : mCurrentLayout(VK_IMAGE_LAYOUT_UNDEFINED)
 {
@@ -704,6 +705,11 @@ Error Framebuffer::init(VkDevice device, const VkFramebufferCreateInfo &createIn
     return NoError();
 }
 
+void Framebuffer::setHandle(VkFramebuffer handle)
+{
+    mHandle = handle;
+}
+
 // DeviceMemory implementation.
 DeviceMemory::DeviceMemory()
 {
@@ -1240,58 +1246,6 @@ void GarbageObject::destroy(VkDevice device)
     }
 }
 
-// CommandBufferAndState implementation.
-CommandBufferAndState::CommandBufferAndState() : mStarted(false)
-{
-}
-
-Error CommandBufferAndState::ensureStarted(VkDevice device,
-                                           const vk::CommandPool &commandPool,
-                                           VkCommandBufferLevel level)
-{
-    ASSERT(commandPool.valid());
-
-    if (valid() && mStarted)
-    {
-        return NoError();
-    }
-
-    if (!valid())
-    {
-        VkCommandBufferAllocateInfo createInfo;
-        createInfo.sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
-        createInfo.pNext              = nullptr;
-        createInfo.commandPool        = commandPool.getHandle();
-        createInfo.level              = level;
-        createInfo.commandBufferCount = 1;
-
-        ANGLE_TRY(init(device, createInfo));
-    }
-    else
-    {
-        reset();
-    }
-
-    VkCommandBufferBeginInfo beginInfo;
-    beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
-    beginInfo.pNext = nullptr;
-    // TODO(jmadill): Use other flags?
-    beginInfo.flags            = 0;
-    beginInfo.pInheritanceInfo = nullptr;
-
-    ANGLE_TRY(begin(beginInfo));
-    mStarted = true;
-
-    return NoError();
-}
-
-Error CommandBufferAndState::ensureFinished()
-{
-    ANGLE_TRY(end());
-    mStarted = false;
-    return NoError();
-}
-
 // RenderPassDesc implementation.
 RenderPassDesc::RenderPassDesc()
 {
@@ -1557,6 +1511,95 @@ VkFrontFace GetFrontFace(GLenum frontFace)
 
 }  // namespace gl_vk
 
+ResourceVk::ResourceVk() : mCurrentWriteNode(nullptr)
+{
+}
+
+ResourceVk::~ResourceVk()
+{
+}
+
+void ResourceVk::updateQueueSerial(Serial queueSerial)
+{
+    ASSERT(queueSerial >= mStoredQueueSerial);
+
+    if (queueSerial > mStoredQueueSerial)
+    {
+        mCurrentWriteNode = nullptr;
+        mCurrentReadNodes.clear();
+        mStoredQueueSerial = queueSerial;
+    }
+}
+
+Serial ResourceVk::getQueueSerial() const
+{
+    return mStoredQueueSerial;
+}
+
+bool ResourceVk::isCurrentlyRecording(Serial currentSerial) const
+{
+    return (mStoredQueueSerial == currentSerial && mCurrentWriteNode != nullptr);
+}
+
+vk::CommandBufferNode *ResourceVk::getCurrentWriteNode(Serial currentSerial)
+{
+    ASSERT(currentSerial == mStoredQueueSerial);
+    return mCurrentWriteNode;
+}
+
+vk::CommandBufferNode *ResourceVk::getNewWriteNode(RendererVk *renderer)
+{
+    vk::CommandBufferNode *newCommands = renderer->allocateCommandNode();
+    setWriteNode(renderer->getCurrentQueueSerial(), newCommands);
+    return newCommands;
+}
+
+void ResourceVk::setWriteNode(Serial serial, vk::CommandBufferNode *newCommands)
+{
+    updateQueueSerial(serial);
+
+    // Make sure any open reads and writes finish before we execute |newCommands|.
+    if (!mCurrentReadNodes.empty())
+    {
+        newCommands->addDependencies(mCurrentReadNodes);
+        mCurrentReadNodes.clear();
+    }
+
+    if (mCurrentWriteNode)
+    {
+        newCommands->addDependency(mCurrentWriteNode);
+    }
+
+    mCurrentWriteNode = newCommands;
+}
+
+vk::Error ResourceVk::recordWriteCommands(RendererVk *renderer,
+                                          vk::CommandBuffer **commandBufferOut)
+{
+    vk::CommandBufferNode *commands = getNewWriteNode(renderer);
+
+    VkDevice device = renderer->getDevice();
+    ANGLE_TRY(commands->startRecording(device, renderer->getCommandPool(), commandBufferOut));
+    return vk::NoError();
+}
+
+void ResourceVk::updateDependencies(vk::CommandBufferNode *readNode, Serial serial)
+{
+    if (isCurrentlyRecording(serial))
+    {
+        // Link the current write node to "readNode".
+        readNode->addDependency(getCurrentWriteNode(serial));
+        ASSERT(mStoredQueueSerial == serial);
+    }
+    else
+    {
+        updateQueueSerial(serial);
+    }
+
+    // Track "readNode" in this resource.
+    mCurrentReadNodes.push_back(readNode);
+}
+
 }  // namespace rx
 
 std::ostream &operator<<(std::ostream &stream, const rx::vk::Error &error)

src/libANGLE/renderer/vulkan/renderervk_utils.h

@@ -49,6 +49,9 @@ ANGLE_GL_OBJECTS_X(ANGLE_PRE_DECLARE_OBJECT);
 namespace rx
 {
 class DisplayVk;
+class RenderTargetVk;
+class RendererVk;
+class ResourceVk;
 
 enum class DrawType
 {
@@ -72,27 +75,9 @@ enum class TextureDimension
     TEX_2D_ARRAY,
 };
 
-// This is a small helper mixin for any GL object used in Vk command buffers. It records a serial
-// at command recording times indicating an order in the queue. We use Fences to detect when
-// commands finish, and then release any unreferenced and deleted resources based on the stored
-// queue serial in a special 'garbage' queue.
-class ResourceVk
-{
-  public:
-    void setQueueSerial(Serial queueSerial)
-    {
-        ASSERT(queueSerial >= mStoredQueueSerial);
-        mStoredQueueSerial = queueSerial;
-    }
-
-    Serial getQueueSerial() const { return mStoredQueueSerial; }
-
-  private:
-    Serial mStoredQueueSerial;
-};
-
 namespace vk
 {
+class CommandBufferNode;
 struct Format;
 
 template <typename T>
@@ -317,6 +302,7 @@ class CommandBuffer : public WrappedObject<CommandBuffer, VkCommandBuffer>
   public:
     CommandBuffer();
 
+    VkCommandBuffer releaseHandle();
     void destroy(VkDevice device, const vk::CommandPool &commandPool);
     Error init(VkDevice device, const VkCommandBufferAllocateInfo &createInfo);
     using WrappedObject::operator=;
@@ -353,11 +339,7 @@ class CommandBuffer : public WrappedObject<CommandBuffer, VkCommandBuffer>
                    uint32_t regionCount,
                    const VkImageCopy *regions);
 
-    void beginRenderPass(const RenderPass &renderPass,
-                         const Framebuffer &framebuffer,
-                         const gl::Rectangle &renderArea,
-                         uint32_t clearValueCount,
-                         const VkClearValue *clearValues);
+    void beginRenderPass(const VkRenderPassBeginInfo &beginInfo, VkSubpassContents subpassContents);
     void endRenderPass();
 
     void draw(uint32_t vertexCount,
@@ -384,6 +366,8 @@ class CommandBuffer : public WrappedObject<CommandBuffer, VkCommandBuffer>
                             const VkDescriptorSet *descriptorSets,
                             uint32_t dynamicOffsetCount,
                             const uint32_t *dynamicOffsets);
+
+    void executeCommands(uint32_t commandBufferCount, const vk::CommandBuffer *commandBuffers);
 };
 
 class Image final : public WrappedObject<Image, VkImage>
@@ -446,6 +430,9 @@ class Framebuffer final : public WrappedObject<Framebuffer, VkFramebuffer>
     Framebuffer();
     void destroy(VkDevice device);
 
+    // Use this method only in necessary cases. (RenderPass)
+    void setHandle(VkFramebuffer handle);
+
     Error init(VkDevice device, const VkFramebufferCreateInfo &createInfo);
 };
 
@@ -662,23 +649,7 @@ struct BufferAndMemory final : private angle::NonCopyable
     vk::DeviceMemory memory;
 };
 
-class CommandBufferAndState : public vk::CommandBuffer
-{
-  public:
-    CommandBufferAndState();
-
-    Error ensureStarted(VkDevice device,
-                        const vk::CommandPool &commandPool,
-                        VkCommandBufferLevel level);
-    Error ensureFinished();
-
-    bool started() const { return mStarted; }
-
-  private:
-    bool mStarted;
-};
-
-using CommandBufferAndSerial = ObjectAndSerial<CommandBufferAndState>;
+using CommandBufferAndSerial = ObjectAndSerial<CommandBuffer>;
 using FenceAndSerial         = ObjectAndSerial<Fence>;
 using RenderPassAndSerial    = ObjectAndSerial<RenderPass>;
 
@@ -771,6 +742,46 @@ VkCullModeFlags GetCullMode(const gl::RasterizerState &rasterState);
 VkFrontFace GetFrontFace(GLenum frontFace);
 }  // namespace gl_vk
 
+// This is a helper class for back-end objects used in Vk command buffers. It records a serial
+// at command recording times indicating an order in the queue. We use Fences to detect when
+// commands finish, and then release any unreferenced and deleted resources based on the stored
+// queue serial in a special 'garbage' queue. Resources also track current read and write
+// dependencies. Only one command buffer node can be writing to the Resource at a time, but many
+// can be reading from it. Together the dependencies will form a command graph at submission time.
+class ResourceVk
+{
+  public:
+    ResourceVk();
+    virtual ~ResourceVk();
+
+    void updateQueueSerial(Serial queueSerial);
+    Serial getQueueSerial() const;
+
+    // Returns true if any tracked read or write nodes match |currentSerial|.
+    bool isCurrentlyRecording(Serial currentSerial) const;
+
+    // Returns the active write node, and asserts |currentSerial| matches the stored serial.
+    vk::CommandBufferNode *getCurrentWriteNode(Serial currentSerial);
+
+    // Allocates a new write node and calls setWriteNode internally.
+    vk::CommandBufferNode *getNewWriteNode(RendererVk *renderer);
+
+    // Called on an operation that will modify this ResourceVk.
+    void setWriteNode(Serial serial, vk::CommandBufferNode *newCommands);
+
+    // Allocates a write node via getNewWriteNode and returns a started command buffer.
+    // The started command buffer will render outside of a RenderPass.
+    vk::Error recordWriteCommands(RendererVk *renderer, vk::CommandBuffer **commandBufferOut);
+
+    // Sets up the dependency relations. |readNode| has the commands that read from this object.
+    void updateDependencies(vk::CommandBufferNode *readNode, Serial serial);
+
+  private:
+    Serial mStoredQueueSerial;
+    std::vector<vk::CommandBufferNode *> mCurrentReadNodes;
+    vk::CommandBufferNode *mCurrentWriteNode;
+};
+
 }  // namespace rx
 
 #define ANGLE_VK_TRY(command)                                          \

src/libGLESv2.gypi

@@ -707,6 +707,8 @@
         [
             'libANGLE/renderer/vulkan/BufferVk.cpp',
             'libANGLE/renderer/vulkan/BufferVk.h',
+            'libANGLE/renderer/vulkan/CommandBufferNode.cpp',
+            'libANGLE/renderer/vulkan/CommandBufferNode.h',
             'libANGLE/renderer/vulkan/CompilerVk.cpp',
             'libANGLE/renderer/vulkan/CompilerVk.h',
             'libANGLE/renderer/vulkan/ContextVk.cpp',