Vulkan: Implement command re-ordering.

src/libANGLE/renderer/vulkan/BufferVk.cpp

@@ -191,11 +191,8 @@ vk::Error BufferVk::setDataImpl(ContextVk *contextVk,
         stagingBuffer.getDeviceMemory().unmap(device);
 
         // Enqueue a copy command on the GPU.
-        // TODO(jmadill): Command re-ordering for render passes.
-        renderer->endRenderPass();
-
-        vk::CommandBufferAndState *commandBuffer = nullptr;
-        ANGLE_TRY(renderer->getStartedCommandBuffer(&commandBuffer));
+        vk::CommandBuffer *commandBuffer = nullptr;
+        ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
         // Insert a barrier to ensure reads from the buffer are complete.
         // TODO(jmadill): Insert minimal barriers.
@@ -216,7 +213,8 @@ vk::Error BufferVk::setDataImpl(ContextVk *contextVk,
         VkBufferCopy copyRegion = {offset, 0, size};
         commandBuffer->copyBuffer(stagingBuffer.getBuffer(), mBuffer, 1, &copyRegion);
 
-        setQueueSerial(renderer->getCurrentQueueSerial());
+        // Immediately release staging buffer.
+        // TODO(jmadill): Staging buffer re-use.
         renderer->releaseObject(getQueueSerial(), &stagingBuffer);
     }
     else

src/libANGLE/renderer/vulkan/CommandBufferNode.h

+//
+// Copyright 2017 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// CommandBufferNode:
+//    Deferred work constructed by GL calls, that will later be flushed to Vulkan.
+//
+
+#ifndef LIBANGLE_RENDERER_VULKAN_COMMAND_BUFFER_NODE_H_
+#define LIBANGLE_RENDERER_VULKAN_COMMAND_BUFFER_NODE_H_
+
+#include "libANGLE/renderer/vulkan/renderervk_utils.h"
+
+namespace rx
+{
+
+namespace vk
+{
+
+enum class VisitedState
+{
+    Unvisited,
+    Ready,
+    Visited,
+};
+
+class CommandBufferNode final : angle::NonCopyable
+{
+  public:
+    CommandBufferNode();
+    ~CommandBufferNode();
+
+    // Immutable queries for when we're walking the commands tree.
+    CommandBuffer *getOutsideRenderPassCommands();
+    CommandBuffer *getInsideRenderPassCommands();
+
+    // For outside the render pass (copies, transitions, etc).
+    Error startRecording(VkDevice device,
+                         const CommandPool &commandPool,
+                         CommandBuffer **commandsOut);
+
+    // For rendering commands (draws).
+    Error startRenderPassRecording(RendererVk *renderer, CommandBuffer **commandsOut);
+
+    // Commands for storing info relevant to the RenderPass.
+    // RenderTargets must be added in order, with the depth/stencil being added last.
+    void storeRenderPassInfo(const Framebuffer &framebuffer,
+                             const gl::Rectangle renderArea,
+                             const std::vector<VkClearValue> &clearValues);
+    void appendColorRenderTarget(Serial serial, RenderTargetVk *colorRenderTarget);
+    void appendDepthStencilRenderTarget(Serial serial, RenderTargetVk *depthStencilRenderTarget);
+
+    // Commands for linking nodes in the dependency graph.
+    void addDependency(CommandBufferNode *node);
+    void addDependencies(const std::vector<CommandBufferNode *> &nodes);
+    bool hasDependencies() const;
+    bool isDependency() const;
+
+    // Used for testing only.
+    bool hasDependency(CommandBufferNode *searchNode);
+
+    // Commands for traversing the node on a flush operation.
+    VisitedState visitedState() const;
+    void visitDependencies(std::vector<CommandBufferNode *> *stack);
+    Error visitAndExecute(RendererVk *renderer, CommandBuffer *primaryCommandBuffer);
+
+  private:
+    void initAttachmentDesc(VkAttachmentDescription *desc);
+    void markAsDependency();
+
+    // Only used if we need a RenderPass for these commands.
+    RenderPassDesc mRenderPassDesc;
+    Framebuffer mRenderPassFramebuffer;
+    gl::Rectangle mRenderPassRenderArea;
+    gl::AttachmentArray<VkClearValue> mRenderPassClearValues;
+
+    // Keep a separate buffers for commands inside and outside a RenderPass.
+    // TODO(jmadill): We might not need inside and outside RenderPass commands separate.
+    CommandBuffer mOutsideRenderPassCommands;
+    CommandBuffer mInsideRenderPassCommands;
+
+    // Dependency commands must finish before these command can execute.
+    std::vector<CommandBufferNode *> mDependencies;
+    bool mIsDependency;
+
+    // Used when traversing the dependency graph.
+    VisitedState mVisitedState;
+};
+}  // namespace vk
+}  // namespace rx
+
+#endif  // LIBANGLE_RENDERER_VULKAN_COMMAND_BUFFER_NODE_H_

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -14,6 +14,7 @@
 #include "libANGLE/Context.h"
 #include "libANGLE/Program.h"
 #include "libANGLE/renderer/vulkan/BufferVk.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/CompilerVk.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
 #include "libANGLE/renderer/vulkan/DeviceVk.h"
@@ -62,7 +63,11 @@ enum DescriptorPoolIndex : uint8_t
 }  // anonymous namespace
 
 ContextVk::ContextVk(const gl::ContextState &state, RendererVk *renderer)
-    : ContextImpl(state), mRenderer(renderer), mCurrentDrawMode(GL_NONE)
+    : ContextImpl(state),
+      mRenderer(renderer),
+      mCurrentDrawMode(GL_NONE),
+      mVertexArrayDirty(false),
+      mTexturesDirty(false)
 {
     // The module handle is filled out at draw time.
     mCurrentShaderStages[0].sType  = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
@@ -238,15 +243,14 @@ gl::Error ContextVk::initialize()
 
 gl::Error ContextVk::flush(const gl::Context *context)
 {
+    // TODO(jmadill): Flush will need to insert a semaphore for the next flush to wait on.
     UNIMPLEMENTED();
     return gl::InternalError();
 }
 
 gl::Error ContextVk::finish(const gl::Context *context)
 {
-    // TODO(jmadill): Implement finish.
-    // UNIMPLEMENTED();
-    return gl::NoError();
+    return mRenderer->finish(context);
 }
 
 gl::Error ContextVk::initPipeline(const gl::Context *context)
@@ -295,7 +299,10 @@ gl::Error ContextVk::initPipeline(const gl::Context *context)
     return gl::NoError();
 }
 
-gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType drawType)
+gl::Error ContextVk::setupDraw(const gl::Context *context,
+                               GLenum mode,
+                               DrawType drawType,
+                               vk::CommandBuffer **commandBuffer)
 {
     if (mode != mCurrentDrawMode)
     {
@@ -325,23 +332,65 @@ gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType
     angle::MemoryBuffer *zeroBuf = nullptr;
     ANGLE_TRY(context->getZeroFilledBuffer(maxAttrib * sizeof(VkDeviceSize), &zeroBuf));
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(mRenderer->getStartedCommandBuffer(&commandBuffer));
-    ANGLE_TRY(mRenderer->ensureInRenderPass(context, vkFBO));
+    // TODO(jmadill): Need to link up the TextureVk to the Secondary CB.
+    vk::CommandBufferNode *renderNode = nullptr;
+    ANGLE_TRY(vkFBO->getRenderNode(context, &renderNode));
+
+    if (!renderNode->getInsideRenderPassCommands()->valid())
+    {
+        mVertexArrayDirty = true;
+        mTexturesDirty    = true;
+        ANGLE_TRY(renderNode->startRenderPassRecording(mRenderer, commandBuffer));
+    }
+    else
+    {
+        *commandBuffer = renderNode->getInsideRenderPassCommands();
+    }
+
+    // Ensure any writes to the VAO buffers are flushed before we read from them.
+    if (mVertexArrayDirty)
+    {
+        mVertexArrayDirty = false;
+        vkVAO->updateDrawDependencies(renderNode, programGL->getActiveAttribLocationsMask(),
+                                      queueSerial, drawType);
+    }
+
+    // Ensure any writes to the textures are flushed before we read from them.
+    if (mTexturesDirty)
+    {
+        mTexturesDirty = false;
+        // TODO(jmadill): Should probably merge this for loop with programVk's descriptor update.
+        const auto &completeTextures = state.getCompleteTextureCache();
+        for (const gl::SamplerBinding &samplerBinding : programGL->getSamplerBindings())
+        {
+            ASSERT(!samplerBinding.unreferenced);
+
+            // TODO(jmadill): Sampler arrays
+            ASSERT(samplerBinding.boundTextureUnits.size() == 1);
+
+            GLuint textureUnit         = samplerBinding.boundTextureUnits[0];
+            const gl::Texture *texture = completeTextures[textureUnit];
 
-    commandBuffer->bindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, mCurrentPipeline);
-    commandBuffer->bindVertexBuffers(0, maxAttrib, vertexHandles.data(),
-                                     reinterpret_cast<const VkDeviceSize *>(zeroBuf->data()));
+            // TODO(jmadill): Incomplete textures handling.
+            ASSERT(texture);
 
+            TextureVk *textureVk = vk::GetImpl(texture);
+            textureVk->updateDependencies(renderNode, mRenderer->getCurrentQueueSerial());
+        }
+    }
+
+    (*commandBuffer)->bindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, mCurrentPipeline);
+    (*commandBuffer)
+        ->bindVertexBuffers(0, maxAttrib, vertexHandles.data(),
+                            reinterpret_cast<const VkDeviceSize *>(zeroBuf->data()));
+
+    // Update the queue serial for the pipeline object.
     // TODO(jmadill): the queue serial should be bound to the pipeline.
-    setQueueSerial(queueSerial);
-    vkVAO->updateCurrentBufferSerials(programGL->getActiveAttribLocationsMask(), queueSerial,
-                                      drawType);
+    updateQueueSerial(queueSerial);
 
     // TODO(jmadill): Can probably use more dirty bits here.
-    ContextVk *contextVk = vk::GetImpl(context);
-    ANGLE_TRY(programVk->updateUniforms(contextVk));
-    programVk->updateTexturesDescriptorSet(contextVk);
+    ANGLE_TRY(programVk->updateUniforms(this));
+    programVk->updateTexturesDescriptorSet(this);
 
     // Bind the graphics descriptor sets.
     // TODO(jmadill): Handle multiple command buffers.
@@ -351,9 +400,9 @@ gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType
     if (!descriptorSets.empty() && ((setCount - firstSet) > 0))
     {
         const vk::PipelineLayout &pipelineLayout = programVk->getPipelineLayout();
-        commandBuffer->bindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, firstSet,
-                                          setCount - firstSet, &descriptorSets[firstSet], 0,
-                                          nullptr);
+        (*commandBuffer)
+            ->bindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, firstSet,
+                                 setCount - firstSet, &descriptorSets[firstSet], 0, nullptr);
     }
 
     return gl::NoError();
@@ -361,11 +410,8 @@ gl::Error ContextVk::setupDraw(const gl::Context *context, GLenum mode, DrawType
 
 gl::Error ContextVk::drawArrays(const gl::Context *context, GLenum mode, GLint first, GLsizei count)
 {
-    ANGLE_TRY(setupDraw(context, mode, DrawType::Arrays));
-
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(mRenderer->getStartedCommandBuffer(&commandBuffer));
-
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(setupDraw(context, mode, DrawType::Arrays, &commandBuffer));
     commandBuffer->draw(count, 1, first, 0);
     return gl::NoError();
 }
@@ -386,7 +432,8 @@ gl::Error ContextVk::drawElements(const gl::Context *context,
                                   GLenum type,
                                   const void *indices)
 {
-    ANGLE_TRY(setupDraw(context, mode, DrawType::Elements));
+    vk::CommandBuffer *commandBuffer;
+    ANGLE_TRY(setupDraw(context, mode, DrawType::Elements, &commandBuffer));
 
     if (indices)
     {
@@ -402,9 +449,6 @@ gl::Error ContextVk::drawElements(const gl::Context *context,
         return gl::InternalError() << "Unsigned byte translation is not yet implemented.";
     }
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(mRenderer->getStartedCommandBuffer(&commandBuffer));
-
     const gl::Buffer *elementArrayBuffer =
         mState.getState().getVertexArray()->getElementArrayBuffer().get();
     ASSERT(elementArrayBuffer);
@@ -444,18 +488,6 @@ VkDevice ContextVk::getDevice() const
     return mRenderer->getDevice();
 }
 
-vk::Error ContextVk::getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut)
-{
-    return mRenderer->getStartedCommandBuffer(commandBufferOut);
-}
-
-vk::Error ContextVk::submitCommands(vk::CommandBufferAndState *commandBuffer)
-{
-    setQueueSerial(mRenderer->getCurrentQueueSerial());
-    ANGLE_TRY(mRenderer->submitCommandBuffer(commandBuffer));
-    return vk::NoError();
-}
-
 gl::Error ContextVk::drawArraysIndirect(const gl::Context *context,
                                         GLenum mode,
                                         const void *indirect)
@@ -681,7 +713,7 @@ void ContextVk::syncState(const gl::Context *context, const gl::State::DirtyBits
                 WARN() << "DIRTY_BIT_RENDERBUFFER_BINDING unimplemented";
                 break;
             case gl::State::DIRTY_BIT_VERTEX_ARRAY_BINDING:
-                WARN() << "DIRTY_BIT_VERTEX_ARRAY_BINDING unimplemented";
+                mVertexArrayDirty = true;
                 break;
             case gl::State::DIRTY_BIT_DRAW_INDIRECT_BUFFER_BINDING:
                 WARN() << "DIRTY_BIT_DRAW_INDIRECT_BUFFER_BINDING unimplemented";
@@ -755,6 +787,7 @@ void ContextVk::syncState(const gl::Context *context, const gl::State::DirtyBits
     {
         ProgramVk *programVk = vk::GetImpl(glState.getProgram());
         programVk->invalidateTextures();
+        mTexturesDirty = true;
     }
 }
 
@@ -875,6 +908,13 @@ void ContextVk::invalidateCurrentPipeline()
     mRenderer->releaseResource(*this, &mCurrentPipeline);
 }
 
+void ContextVk::onVertexArrayChange()
+{
+    // TODO(jmadill): Does not handle dependent state changes.
+    mVertexArrayDirty = true;
+    invalidateCurrentPipeline();
+}
+
 gl::Error ContextVk::dispatchCompute(const gl::Context *context,
                                      GLuint numGroupsX,
                                      GLuint numGroupsY,

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -140,15 +140,6 @@ class ContextVk : public ContextImpl, public ResourceVk
     // Path object creation
     std::vector<PathImpl *> createPaths(GLsizei) override;
 
-    VkDevice getDevice() const;
-    vk::Error getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut);
-    vk::Error submitCommands(vk::CommandBufferAndState *commandBuffer);
-
-    RendererVk *getRenderer() { return mRenderer; }
-
-    // TODO(jmadill): Use pipeline cache.
-    void invalidateCurrentPipeline();
-
     gl::Error dispatchCompute(const gl::Context *context,
                               GLuint numGroupsX,
                               GLuint numGroupsY,
@@ -158,11 +149,22 @@ class ContextVk : public ContextImpl, public ResourceVk
     gl::Error memoryBarrier(const gl::Context *context, GLbitfield barriers) override;
     gl::Error memoryBarrierByRegion(const gl::Context *context, GLbitfield barriers) override;
 
+    VkDevice getDevice() const;
+    RendererVk *getRenderer() { return mRenderer; }
+
+    // TODO(jmadill): Use pipeline cache.
+    void invalidateCurrentPipeline();
+
+    void onVertexArrayChange();
+
     vk::DescriptorPool *getDescriptorPool();
 
   private:
     gl::Error initPipeline(const gl::Context *context);
-    gl::Error setupDraw(const gl::Context *context, GLenum mode, DrawType drawType);
+    gl::Error setupDraw(const gl::Context *context,
+                        GLenum mode,
+                        DrawType drawType,
+                        vk::CommandBuffer **commandBuffer);
 
     RendererVk *mRenderer;
     vk::Pipeline mCurrentPipeline;
@@ -187,6 +189,10 @@ class ContextVk : public ContextImpl, public ResourceVk
     // The descriptor pool is externally sychronized, so cannot be accessed from different threads
     // simulataneously. Hence, we keep it in the ContextVk instead of the RendererVk.
     vk::DescriptorPool mDescriptorPool;
+
+    // Triggers adding dependencies to the command graph.
+    bool mVertexArrayDirty;
+    bool mTexturesDirty;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/FramebufferVk.h

@@ -84,12 +84,8 @@ class FramebufferVk : public FramebufferImpl, public ResourceVk
 
     gl::Error getSamplePosition(size_t index, GLfloat *xy) const override;
 
-    gl::Error beginRenderPass(const gl::Context *context,
-                              RendererVk *rendererVk,
-                              vk::CommandBuffer *commandBuffer,
-                              Serial queueSerial);
-
     const vk::RenderPassDesc &getRenderPassDesc(const gl::Context *context);
+    gl::Error getRenderNode(const gl::Context *context, vk::CommandBufferNode **nodeOut);
 
   private:
     FramebufferVk(const gl::FramebufferState &state);
@@ -102,6 +98,7 @@ class FramebufferVk : public FramebufferImpl, public ResourceVk
 
     Optional<vk::RenderPassDesc> mRenderPassDesc;
     vk::Framebuffer mFramebuffer;
+    bool mRenderNodeDirty;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -79,14 +79,12 @@ class RendererVk : angle::NonCopyable
 
     vk::ErrorOrResult<uint32_t> selectPresentQueueForSurface(VkSurfaceKHR surface);
 
-    // TODO(jmadill): Use ContextImpl for command buffers to enable threaded contexts.
-    vk::Error getStartedCommandBuffer(vk::CommandBufferAndState **commandBufferOut);
-    vk::Error submitCommandBuffer(vk::CommandBufferAndState *commandBuffer);
-    vk::Error submitAndFinishCommandBuffer(vk::CommandBufferAndState *commandBuffer);
-    vk::Error submitCommandsWithSync(vk::CommandBufferAndState *commandBuffer,
-                                     const vk::Semaphore &waitSemaphore,
-                                     const vk::Semaphore &signalSemaphore);
-    vk::Error finish();
+    vk::Error finish(const gl::Context *context);
+    vk::Error flush(const gl::Context *context,
+                    const vk::Semaphore &waitSemaphore,
+                    const vk::Semaphore &signalSemaphore);
+
+    const vk::CommandPool &getCommandPool() const;
 
     const gl::Caps &getNativeCaps() const;
     const gl::TextureCapsMap &getNativeTextureCaps() const;
@@ -130,13 +128,6 @@ class RendererVk : angle::NonCopyable
 
     const vk::MemoryProperties &getMemoryProperties() const { return mMemoryProperties; }
 
-    // TODO(jmadill): Don't keep a single renderpass in the Renderer.
-    gl::Error ensureInRenderPass(const gl::Context *context, FramebufferVk *framebufferVk);
-    void endRenderPass();
-
-    // This is necessary to update the cached current RenderPass Framebuffer.
-    void onReleaseRenderPass(const FramebufferVk *framebufferVk);
-
     // TODO(jmadill): We could pass angle::Format::ID here.
     const vk::Format &getFormat(GLenum internalFormat) const
     {
@@ -149,16 +140,22 @@ class RendererVk : angle::NonCopyable
                                    const vk::AttachmentOpsArray &ops,
                                    vk::RenderPass **renderPassOut);
 
+    // This should only be called from ResourceVk.
+    // TODO(jmadill): Keep in ContextVk to enable threaded rendering.
+    vk::CommandBufferNode *allocateCommandNode();
+
   private:
+    vk::Error initializeDevice(uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
     void generateCaps(gl::Caps *outCaps,
                       gl::TextureCapsMap *outTextureCaps,
                       gl::Extensions *outExtensions,
                       gl::Limitations *outLimitations) const;
-    vk::Error submit(const VkSubmitInfo &submitInfo);
-    vk::Error submitFrame(const VkSubmitInfo &submitInfo);
+    vk::Error submitFrame(const VkSubmitInfo &submitInfo, vk::CommandBuffer &&commandBatch);
     vk::Error checkInFlightCommands();
     void freeAllInFlightResources();
+    vk::Error flushCommandGraph(const gl::Context *context, vk::CommandBuffer *commandBatch);
+    void resetCommandGraph();
 
     mutable bool mCapsInitialized;
     mutable gl::Caps mNativeCaps;
@@ -166,8 +163,6 @@ class RendererVk : angle::NonCopyable
     mutable gl::Extensions mNativeExtensions;
     mutable gl::Limitations mNativeLimitations;
 
-    vk::Error initializeDevice(uint32_t queueFamilyIndex);
-
     VkInstance mInstance;
     bool mEnableValidationLayers;
     VkDebugReportCallbackEXT mDebugReportCallback;
@@ -178,21 +173,30 @@ class RendererVk : angle::NonCopyable
     uint32_t mCurrentQueueFamilyIndex;
     VkDevice mDevice;
     vk::CommandPool mCommandPool;
-    vk::CommandBufferAndState mCommandBuffer;
     GlslangWrapper *mGlslangWrapper;
     SerialFactory mQueueSerialFactory;
     Serial mLastCompletedQueueSerial;
     Serial mCurrentQueueSerial;
-    std::vector<vk::CommandBufferAndSerial> mInFlightCommands;
-    std::vector<vk::FenceAndSerial> mInFlightFences;
+
+    struct CommandBatch final : angle::NonCopyable
+    {
+        CommandBatch();
+        ~CommandBatch();
+        CommandBatch(CommandBatch &&other);
+        CommandBatch &operator=(CommandBatch &&other);
+
+        vk::CommandPool commandPool;
+        vk::Fence fence;
+        Serial serial;
+    };
+
+    std::vector<CommandBatch> mInFlightCommands;
     std::vector<vk::GarbageObject> mGarbage;
     vk::MemoryProperties mMemoryProperties;
     vk::FormatTable mFormatTable;
 
-    // TODO(jmadill): Don't keep a single renderpass in the Renderer.
-    FramebufferVk *mCurrentRenderPassFramebuffer;
-
     RenderPassCache mRenderPassCache;
+    std::vector<vk::CommandBufferNode *> mOpenCommandGraph;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -192,7 +192,7 @@ void WindowSurfaceVk::destroy(const egl::Display *display)
     VkDevice device            = rendererVk->getDevice();
     VkInstance instance        = rendererVk->getInstance();
 
-    rendererVk->finish();
+    rendererVk->finish(display->getProxyContext());
 
     mAcquireNextImageSemaphore.destroy(device);
 
@@ -359,9 +359,9 @@ vk::Error WindowSurfaceVk::initializeImpl(RendererVk *renderer)
     std::vector<VkImage> swapchainImages(imageCount);
     ANGLE_VK_TRY(vkGetSwapchainImagesKHR(device, mSwapchain, &imageCount, swapchainImages.data()));
 
-    // CommandBuffer is a singleton in the Renderer.
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(renderer->getStartedCommandBuffer(&commandBuffer));
+    // Allocate a command buffer for clearing our images to black.
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
     VkClearColorValue transparentBlack;
     transparentBlack.float32[0] = 0.0f;
@@ -409,8 +409,6 @@ vk::Error WindowSurfaceVk::initializeImpl(RendererVk *renderer)
         ANGLE_TRY(member.commandsCompleteSemaphore.init(device));
     }
 
-    ANGLE_TRY(renderer->submitAndFinishCommandBuffer(commandBuffer));
-
     // Get the first available swapchain iamge.
     ANGLE_TRY(nextSwapchainImage(renderer));
 
@@ -427,20 +425,17 @@ egl::Error WindowSurfaceVk::swap(const gl::Context *context)
     const DisplayVk *displayVk = vk::GetImpl(context->getCurrentDisplay());
     RendererVk *renderer       = displayVk->getRenderer();
 
-    vk::CommandBufferAndState *currentCB = nullptr;
-    ANGLE_TRY(renderer->getStartedCommandBuffer(&currentCB));
-
-    // End render pass
-    renderer->endRenderPass();
+    vk::CommandBuffer *swapCommands = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &swapCommands));
 
     auto &image = mSwapchainImages[mCurrentSwapchainImageIndex];
 
     image.image.changeLayoutWithStages(VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
                                        VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
-                                       VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, currentCB);
+                                       VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, swapCommands);
 
-    ANGLE_TRY(renderer->submitCommandsWithSync(currentCB, image.imageAcquiredSemaphore,
-                                               image.commandsCompleteSemaphore));
+    ANGLE_TRY(
+        renderer->flush(context, image.imageAcquiredSemaphore, image.commandsCompleteSemaphore));
 
     VkPresentInfoKHR presentInfo;
     presentInfo.sType              = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;

src/libANGLE/renderer/vulkan/TextureVk.cpp

@@ -177,7 +177,6 @@ gl::Error TextureVk::setImage(const gl::Context *context,
     mRenderTarget.resource  = this;
 
     // Handle initial data.
-    // TODO(jmadill): Consider re-using staging texture.
     if (pixels)
     {
         ANGLE_TRY(setSubImageImpl(contextVk, formatInfo, unpack, type, pixels));
@@ -258,13 +257,8 @@ gl::Error TextureVk::setSubImageImpl(ContextVk *contextVk,
 
     stagingImage.getDeviceMemory().unmap(device);
 
-    vk::CommandBufferAndState *commandBuffer = nullptr;
-    ANGLE_TRY(contextVk->getStartedCommandBuffer(&commandBuffer));
-    setQueueSerial(renderer->getCurrentQueueSerial());
-
-    // Ensure we aren't in a render pass.
-    // TODO(jmadill): Command reordering.
-    renderer->endRenderPass();
+    vk::CommandBuffer *commandBuffer = nullptr;
+    ANGLE_TRY(recordWriteCommands(renderer, &commandBuffer));
 
     stagingImage.getImage().changeLayoutWithStages(
         VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
@@ -277,7 +271,8 @@ gl::Error TextureVk::setSubImageImpl(ContextVk *contextVk,
     commandBuffer->copySingleImage(stagingImage.getImage(), mImage, wholeRegion,
                                    VK_IMAGE_ASPECT_COLOR_BIT);
 
-    // TODO(jmadill): Re-use staging images.
+    // Immediately release staging image.
+    // TODO(jmadill): Staging image re-use.
     renderer->releaseObject(renderer->getCurrentQueueSerial(), &stagingImage);
     return gl::NoError();
 }

src/libANGLE/renderer/vulkan/VertexArrayVk.cpp

@@ -13,6 +13,7 @@
 
 #include "libANGLE/Context.h"
 #include "libANGLE/renderer/vulkan/BufferVk.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
 #include "libANGLE/renderer/vulkan/formatutilsvk.h"
 
@@ -48,7 +49,7 @@ void VertexArrayVk::syncState(const gl::Context *context,
     // Invalidate current pipeline.
     // TODO(jmadill): Use pipeline cache.
     ContextVk *contextVk = vk::GetImpl(context);
-    contextVk->invalidateCurrentPipeline();
+    contextVk->onVertexArrayChange();
 
     // Invalidate the vertex descriptions.
     invalidateVertexDescriptions();
@@ -107,22 +108,23 @@ const gl::AttribArray<VkBuffer> &VertexArrayVk::getCurrentArrayBufferHandles() c
     return mCurrentArrayBufferHandles;
 }
 
-void VertexArrayVk::updateCurrentBufferSerials(const gl::AttributesMask &activeAttribsMask,
-                                               Serial serial,
-                                               DrawType drawType)
+void VertexArrayVk::updateDrawDependencies(vk::CommandBufferNode *readNode,
+                                           const gl::AttributesMask &activeAttribsMask,
+                                           Serial serial,
+                                           DrawType drawType)
 {
     // Handle the bound array buffers.
     for (auto attribIndex : activeAttribsMask)
     {
         ASSERT(mCurrentArrayBufferResources[attribIndex]);
-        mCurrentArrayBufferResources[attribIndex]->setQueueSerial(serial);
+        mCurrentArrayBufferResources[attribIndex]->updateDependencies(readNode, serial);
     }
 
     // Handle the bound element array buffer.
     if (drawType == DrawType::Elements)
     {
         ASSERT(mCurrentElementArrayBufferResource);
-        mCurrentElementArrayBufferResource->setQueueSerial(serial);
+        mCurrentElementArrayBufferResource->updateDependencies(readNode, serial);
     }
 }
 

src/libANGLE/renderer/vulkan/VertexArrayVk.h

@@ -30,9 +30,10 @@ class VertexArrayVk : public VertexArrayImpl
 
     const gl::AttribArray<VkBuffer> &getCurrentArrayBufferHandles() const;
 
-    void updateCurrentBufferSerials(const gl::AttributesMask &activeAttribsMask,
-                                    Serial serial,
-                                    DrawType drawType);
+    void updateDrawDependencies(vk::CommandBufferNode *readNode,
+                                const gl::AttributesMask &activeAttribsMask,
+                                Serial serial,
+                                DrawType drawType);
 
     void invalidateVertexDescriptions();
     void updateVertexDescriptions(const gl::Context *context);

src/libANGLE/renderer/vulkan/renderervk_utils.cpp

@@ -9,8 +9,11 @@
 
 #include "renderervk_utils.h"
 
+#include "libANGLE/Context.h"
 #include "libANGLE/SizedMRUCache.h"
+#include "libANGLE/renderer/vulkan/CommandBufferNode.h"
 #include "libANGLE/renderer/vulkan/ContextVk.h"
+#include "libANGLE/renderer/vulkan/RenderTargetVk.h"
 #include "libANGLE/renderer/vulkan/RendererVk.h"
 
 namespace rx
@@ -315,6 +318,20 @@ CommandBuffer::CommandBuffer()
 {
 }
 
+VkCommandBuffer CommandBuffer::releaseHandle()
+{
+    VkCommandBuffer handle = mHandle;
+    mHandle                = nullptr;
+    return handle;
+}
+
+Error CommandBuffer::init(VkDevice device, const VkCommandBufferAllocateInfo &createInfo)
+{
+    ASSERT(!valid());
+    ANGLE_VK_TRY(vkAllocateCommandBuffers(device, &createInfo, &mHandle));
+    return NoError();
+}
+
 Error CommandBuffer::begin(const VkCommandBufferBeginInfo &info)
 {
     ASSERT(valid());
@@ -366,13 +383,6 @@ void CommandBuffer::destroy(VkDevice device, const vk::CommandPool &commandPool)
     }
 }
 
-Error CommandBuffer::init(VkDevice device, const VkCommandBufferAllocateInfo &createInfo)
-{
-    ASSERT(!valid());
-    ANGLE_VK_TRY(vkAllocateCommandBuffers(device, &createInfo, &mHandle));
-    return NoError();
-}
-
 void CommandBuffer::copyBuffer(const vk::Buffer &srcBuffer,
                                const vk::Buffer &destBuffer,
                                uint32_t regionCount,
@@ -440,27 +450,11 @@ void CommandBuffer::copyImage(const vk::Image &srcImage,
                    dstImage.getCurrentLayout(), 1, regions);
 }
 
-void CommandBuffer::beginRenderPass(const RenderPass &renderPass,
-                                    const Framebuffer &framebuffer,
-                                    const gl::Rectangle &renderArea,
-                                    uint32_t clearValueCount,
-                                    const VkClearValue *clearValues)
+void CommandBuffer::beginRenderPass(const VkRenderPassBeginInfo &beginInfo,
+                                    VkSubpassContents subpassContents)
 {
-    ASSERT(mHandle != VK_NULL_HANDLE);
-
-    VkRenderPassBeginInfo beginInfo;
-    beginInfo.sType                    = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
-    beginInfo.pNext                    = nullptr;
-    beginInfo.renderPass               = renderPass.getHandle();
-    beginInfo.framebuffer              = framebuffer.getHandle();
-    beginInfo.renderArea.offset.x      = static_cast<uint32_t>(renderArea.x);
-    beginInfo.renderArea.offset.y      = static_cast<uint32_t>(renderArea.y);
-    beginInfo.renderArea.extent.width  = static_cast<uint32_t>(renderArea.width);
-    beginInfo.renderArea.extent.height = static_cast<uint32_t>(renderArea.height);
-    beginInfo.clearValueCount          = clearValueCount;
-    beginInfo.pClearValues             = clearValues;
-
-    vkCmdBeginRenderPass(mHandle, &beginInfo, VK_SUBPASS_CONTENTS_INLINE);
+    ASSERT(valid());
+    vkCmdBeginRenderPass(mHandle, &beginInfo, subpassContents);
 }
 
 void CommandBuffer::endRenderPass()
@@ -525,6 +519,13 @@ void CommandBuffer::bindDescriptorSets(VkPipelineBindPoint bindPoint,
                             descriptorSets, dynamicOffsetCount, dynamicOffsets);
 }
 
+void CommandBuffer::executeCommands(uint32_t commandBufferCount,
+                                    const vk::CommandBuffer *commandBuffers)
+{
+    ASSERT(valid());
+    vkCmdExecuteCommands(mHandle, commandBufferCount, commandBuffers[0].ptr());
+}
+
 // Image implementation.
 Image::Image() : mCurrentLayout(VK_IMAGE_LAYOUT_UNDEFINED)
 {
@@ -704,6 +705,11 @@ Error Framebuffer::init(VkDevice device, const VkFramebufferCreateInfo &createIn
     return NoError();
 }
 
+void Framebuffer::setHandle(VkFramebuffer handle)
+{
+    mHandle = handle;
+}
+
 // DeviceMemory implementation.
 DeviceMemory::DeviceMemory()
 {
@@ -1240,58 +1246,6 @@ void GarbageObject::destroy(VkDevice device)
     }
 }
 
-// CommandBufferAndState implementation.
-CommandBufferAndState::CommandBufferAndState() : mStarted(false)
-{
-}
-
-Error CommandBufferAndState::ensureStarted(VkDevice device,
-                                           const vk::CommandPool &commandPool,
-                                           VkCommandBufferLevel level)
-{
-    ASSERT(commandPool.valid());
-
-    if (valid() && mStarted)
-    {
-        return NoError();
-    }
-
-    if (!valid())
-    {
-        VkCommandBufferAllocateInfo createInfo;
-        createInfo.sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
-        createInfo.pNext              = nullptr;
-        createInfo.commandPool        = commandPool.getHandle();
-        createInfo.level              = level;
-        createInfo.commandBufferCount = 1;
-
-        ANGLE_TRY(init(device, createInfo));
-    }
-    else
-    {
-        reset();
-    }
-
-    VkCommandBufferBeginInfo beginInfo;
-    beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
-    beginInfo.pNext = nullptr;
-    // TODO(jmadill): Use other flags?
-    beginInfo.flags            = 0;
-    beginInfo.pInheritanceInfo = nullptr;
-
-    ANGLE_TRY(begin(beginInfo));
-    mStarted = true;
-
-    return NoError();
-}
-
-Error CommandBufferAndState::ensureFinished()
-{
-    ANGLE_TRY(end());
-    mStarted = false;
-    return NoError();
-}
-
 // RenderPassDesc implementation.
 RenderPassDesc::RenderPassDesc()
 {
@@ -1557,6 +1511,95 @@ VkFrontFace GetFrontFace(GLenum frontFace)
 
 }  // namespace gl_vk
 
+ResourceVk::ResourceVk() : mCurrentWriteNode(nullptr)
+{
+}
+
+ResourceVk::~ResourceVk()
+{
+}
+
+void ResourceVk::updateQueueSerial(Serial queueSerial)
+{
+    ASSERT(queueSerial >= mStoredQueueSerial);
+
+    if (queueSerial > mStoredQueueSerial)
+    {
+        mCurrentWriteNode = nullptr;
+        mCurrentReadNodes.clear();
+        mStoredQueueSerial = queueSerial;
+    }
+}
+
+Serial ResourceVk::getQueueSerial() const
+{
+    return mStoredQueueSerial;
+}
+
+bool ResourceVk::isCurrentlyRecording(Serial currentSerial) const
+{
+    return (mStoredQueueSerial == currentSerial && mCurrentWriteNode != nullptr);
+}
+
+vk::CommandBufferNode *ResourceVk::getCurrentWriteNode(Serial currentSerial)
+{
+    ASSERT(currentSerial == mStoredQueueSerial);
+    return mCurrentWriteNode;
+}
+
+vk::CommandBufferNode *ResourceVk::getNewWriteNode(RendererVk *renderer)
+{
+    vk::CommandBufferNode *newCommands = renderer->allocateCommandNode();
+    setWriteNode(renderer->getCurrentQueueSerial(), newCommands);
+    return newCommands;
+}
+
+void ResourceVk::setWriteNode(Serial serial, vk::CommandBufferNode *newCommands)
+{
+    updateQueueSerial(serial);
+
+    // Make sure any open reads and writes finish before we execute |newCommands|.
+    if (!mCurrentReadNodes.empty())
+    {
+        newCommands->addDependencies(mCurrentReadNodes);
+        mCurrentReadNodes.clear();
+    }
+
+    if (mCurrentWriteNode)
+    {
+        newCommands->addDependency(mCurrentWriteNode);
+    }
+
+    mCurrentWriteNode = newCommands;
+}
+
+vk::Error ResourceVk::recordWriteCommands(RendererVk *renderer,
+                                          vk::CommandBuffer **commandBufferOut)
+{
+    vk::CommandBufferNode *commands = getNewWriteNode(renderer);
+
+    VkDevice device = renderer->getDevice();
+    ANGLE_TRY(commands->startRecording(device, renderer->getCommandPool(), commandBufferOut));
+    return vk::NoError();
+}
+
+void ResourceVk::updateDependencies(vk::CommandBufferNode *readNode, Serial serial)
+{
+    if (isCurrentlyRecording(serial))
+    {
+        // Link the current write node to "readNode".
+        readNode->addDependency(getCurrentWriteNode(serial));
+        ASSERT(mStoredQueueSerial == serial);
+    }
+    else
+    {
+        updateQueueSerial(serial);
+    }
+
+    // Track "readNode" in this resource.
+    mCurrentReadNodes.push_back(readNode);
+}
+
 }  // namespace rx
 
 std::ostream &operator<<(std::ostream &stream, const rx::vk::Error &error)

src/libANGLE/renderer/vulkan/renderervk_utils.h

@@ -49,6 +49,9 @@ ANGLE_GL_OBJECTS_X(ANGLE_PRE_DECLARE_OBJECT);
 namespace rx
 {
 class DisplayVk;
+class RenderTargetVk;
+class RendererVk;
+class ResourceVk;
 
 enum class DrawType
 {
@@ -72,27 +75,9 @@ enum class TextureDimension
     TEX_2D_ARRAY,
 };
 
-// This is a small helper mixin for any GL object used in Vk command buffers. It records a serial
-// at command recording times indicating an order in the queue. We use Fences to detect when
-// commands finish, and then release any unreferenced and deleted resources based on the stored
-// queue serial in a special 'garbage' queue.
-class ResourceVk
-{
-  public:
-    void setQueueSerial(Serial queueSerial)
-    {
-        ASSERT(queueSerial >= mStoredQueueSerial);
-        mStoredQueueSerial = queueSerial;
-    }
-
-    Serial getQueueSerial() const { return mStoredQueueSerial; }
-
-  private:
-    Serial mStoredQueueSerial;
-};
-
 namespace vk
 {
+class CommandBufferNode;
 struct Format;
 
 template <typename T>
@@ -317,6 +302,7 @@ class CommandBuffer : public WrappedObject<CommandBuffer, VkCommandBuffer>
   public:
     CommandBuffer();
 
+    VkCommandBuffer releaseHandle();
     void destroy(VkDevice device, const vk::CommandPool &commandPool);
     Error init(VkDevice device, const VkCommandBufferAllocateInfo &createInfo);
     using WrappedObject::operator=;
@@ -353,11 +339,7 @@ class CommandBuffer : public WrappedObject<CommandBuffer, VkCommandBuffer>
                    uint32_t regionCount,
                    const VkImageCopy *regions);
 
-    void beginRenderPass(const RenderPass &renderPass,
-                         const Framebuffer &framebuffer,
-                         const gl::Rectangle &renderArea,
-                         uint32_t clearValueCount,
-                         const VkClearValue *clearValues);
+    void beginRenderPass(const VkRenderPassBeginInfo &beginInfo, VkSubpassContents subpassContents);
     void endRenderPass();
 
     void draw(uint32_t vertexCount,
@@ -384,6 +366,8 @@ class CommandBuffer : public WrappedObject<CommandBuffer, VkCommandBuffer>
                             const VkDescriptorSet *descriptorSets,
                             uint32_t dynamicOffsetCount,
                             const uint32_t *dynamicOffsets);
+
+    void executeCommands(uint32_t commandBufferCount, const vk::CommandBuffer *commandBuffers);
 };
 
 class Image final : public WrappedObject<Image, VkImage>
@@ -446,6 +430,9 @@ class Framebuffer final : public WrappedObject<Framebuffer, VkFramebuffer>
     Framebuffer();
     void destroy(VkDevice device);
 
+    // Use this method only in necessary cases. (RenderPass)
+    void setHandle(VkFramebuffer handle);
+
     Error init(VkDevice device, const VkFramebufferCreateInfo &createInfo);
 };
 
@@ -662,23 +649,7 @@ struct BufferAndMemory final : private angle::NonCopyable
     vk::DeviceMemory memory;
 };
 
-class CommandBufferAndState : public vk::CommandBuffer
-{
-  public:
-    CommandBufferAndState();
-
-    Error ensureStarted(VkDevice device,
-                        const vk::CommandPool &commandPool,
-                        VkCommandBufferLevel level);
-    Error ensureFinished();
-
-    bool started() const { return mStarted; }
-
-  private:
-    bool mStarted;
-};
-
-using CommandBufferAndSerial = ObjectAndSerial<CommandBufferAndState>;
+using CommandBufferAndSerial = ObjectAndSerial<CommandBuffer>;
 using FenceAndSerial         = ObjectAndSerial<Fence>;
 using RenderPassAndSerial    = ObjectAndSerial<RenderPass>;
 
@@ -771,6 +742,46 @@ VkCullModeFlags GetCullMode(const gl::RasterizerState &rasterState);
 VkFrontFace GetFrontFace(GLenum frontFace);
 }  // namespace gl_vk
 
+// This is a helper class for back-end objects used in Vk command buffers. It records a serial
+// at command recording times indicating an order in the queue. We use Fences to detect when
+// commands finish, and then release any unreferenced and deleted resources based on the stored
+// queue serial in a special 'garbage' queue. Resources also track current read and write
+// dependencies. Only one command buffer node can be writing to the Resource at a time, but many
+// can be reading from it. Together the dependencies will form a command graph at submission time.
+class ResourceVk
+{
+  public:
+    ResourceVk();
+    virtual ~ResourceVk();
+
+    void updateQueueSerial(Serial queueSerial);
+    Serial getQueueSerial() const;
+
+    // Returns true if any tracked read or write nodes match |currentSerial|.
+    bool isCurrentlyRecording(Serial currentSerial) const;
+
+    // Returns the active write node, and asserts |currentSerial| matches the stored serial.
+    vk::CommandBufferNode *getCurrentWriteNode(Serial currentSerial);
+
+    // Allocates a new write node and calls setWriteNode internally.
+    vk::CommandBufferNode *getNewWriteNode(RendererVk *renderer);
+
+    // Called on an operation that will modify this ResourceVk.
+    void setWriteNode(Serial serial, vk::CommandBufferNode *newCommands);
+
+    // Allocates a write node via getNewWriteNode and returns a started command buffer.
+    // The started command buffer will render outside of a RenderPass.
+    vk::Error recordWriteCommands(RendererVk *renderer, vk::CommandBuffer **commandBufferOut);
+
+    // Sets up the dependency relations. |readNode| has the commands that read from this object.
+    void updateDependencies(vk::CommandBufferNode *readNode, Serial serial);
+
+  private:
+    Serial mStoredQueueSerial;
+    std::vector<vk::CommandBufferNode *> mCurrentReadNodes;
+    vk::CommandBufferNode *mCurrentWriteNode;
+};
+
 }  // namespace rx
 
 #define ANGLE_VK_TRY(command)                                          \

src/libGLESv2.gypi

@@ -707,6 +707,8 @@
         [
             'libANGLE/renderer/vulkan/BufferVk.cpp',
             'libANGLE/renderer/vulkan/BufferVk.h',
+            'libANGLE/renderer/vulkan/CommandBufferNode.cpp',
+            'libANGLE/renderer/vulkan/CommandBufferNode.h',
             'libANGLE/renderer/vulkan/CompilerVk.cpp',
             'libANGLE/renderer/vulkan/CompilerVk.h',
             'libANGLE/renderer/vulkan/ContextVk.cpp',