Vulkan: Enable command queueing.

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -280,7 +280,7 @@ gl::Error ContextVk::drawArrays(GLenum mode, GLint first, GLsizei count)
     commandBuffer->endRenderPass();
     ANGLE_TRY(commandBuffer->end());
 
-    ANGLE_TRY(mRenderer->submitAndFinishCommandBuffer(*commandBuffer));
+    ANGLE_TRY(mRenderer->submitCommandBuffer(*commandBuffer));
 
     return gl::NoError();
 }
@@ -338,8 +338,7 @@ vk::CommandBuffer *ContextVk::getCommandBuffer()
 
 vk::Error ContextVk::submitCommands(const vk::CommandBuffer &commandBuffer)
 {
-    // TODO(jmadill): Command queuing.
-    ANGLE_TRY(mRenderer->submitAndFinishCommandBuffer(commandBuffer));
+    ANGLE_TRY(mRenderer->submitCommandBuffer(commandBuffer));
     return vk::NoError();
 }
 

src/libANGLE/renderer/vulkan/FramebufferVk.cpp

@@ -322,8 +322,10 @@ gl::Error FramebufferVk::readPixels(ContextImpl *context,
 
     PackPixels(params, angleFormat, inputPitch, mapPointer, reinterpret_cast<uint8_t *>(pixels));
 
-    stagingImage.getImage().destroy(renderer->getDevice());
     stagingImage.getDeviceMemory().unmap(device);
+    renderer->enqueueGarbage(renderer->getCurrentQueueSerial(), std::move(stagingImage));
+
+    stagingImage.getImage().destroy(renderer->getDevice());
 
     stagingImage.destroy(device);
 

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -576,7 +576,7 @@ vk::CommandBuffer *RendererVk::getCommandBuffer()
     return &mCommandBuffer;
 }
 
-vk::Error RendererVk::submitAndFinishCommandBuffer(const vk::CommandBuffer &commandBuffer)
+vk::Error RendererVk::submitCommandBuffer(const vk::CommandBuffer &commandBuffer)
 {
     VkFenceCreateInfo fenceInfo;
     fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
@@ -599,7 +599,12 @@ vk::Error RendererVk::submitAndFinishCommandBuffer(const vk::CommandBuffer &comm
     // TODO(jmadill): Investigate how to properly submit command buffers.
     ANGLE_TRY(submit(submitInfo));
 
-    // Wait indefinitely for the queue to finish.
+    return vk::NoError();
+}
+
+vk::Error RendererVk::submitAndFinishCommandBuffer(const vk::CommandBuffer &commandBuffer)
+{
+    ANGLE_TRY(submitCommandBuffer(commandBuffer));
     ANGLE_TRY(finish());
 
     return vk::NoError();
@@ -635,13 +640,15 @@ vk::Error RendererVk::waitThenFinishCommandBuffer(const vk::CommandBuffer &comma
 vk::Error RendererVk::finish()
 {
     ASSERT(mQueue != VK_NULL_HANDLE);
-    checkInFlightCommands();
     ANGLE_VK_TRY(vkQueueWaitIdle(mQueue));
+    checkInFlightCommands();
     return vk::NoError();
 }
 
 vk::Error RendererVk::checkInFlightCommands()
 {
+    bool anyFinished = false;
+
     // Check if any in-flight command buffers are finished.
     for (size_t index = 0; index < mInFlightCommands.size();)
     {
@@ -655,6 +662,7 @@ vk::Error RendererVk::checkInFlightCommands()
             mLastCompletedQueueSerial = inFlightCommand->queueSerial();
             inFlightCommand->destroy(mDevice);
             mInFlightCommands.erase(mInFlightCommands.begin() + index);
+            anyFinished = true;
         }
         else
         {
@@ -662,6 +670,22 @@ vk::Error RendererVk::checkInFlightCommands()
         }
     }
 
+    if (anyFinished)
+    {
+        size_t freeIndex = 0;
+        for (; freeIndex < mGarbage.size(); ++freeIndex)
+        {
+            if (!mGarbage[freeIndex]->destroyIfComplete(mDevice, mLastCompletedQueueSerial))
+                break;
+        }
+
+        // Remove the entries from the garbage list - they should be ready to go.
+        if (freeIndex > 0)
+        {
+            mGarbage.erase(mGarbage.begin(), mGarbage.begin() + freeIndex);
+        }
+    }
+
     return vk::NoError();
 }
 

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -51,6 +51,7 @@ class RendererVk : angle::NonCopyable
 
     // TODO(jmadill): Use ContextImpl for command buffers to enable threaded contexts.
     vk::CommandBuffer *getCommandBuffer();
+    vk::Error submitCommandBuffer(const vk::CommandBuffer &commandBuffer);
     vk::Error submitAndFinishCommandBuffer(const vk::CommandBuffer &commandBuffer);
     vk::Error waitThenFinishCommandBuffer(const vk::CommandBuffer &commandBuffer,
                                           const vk::Semaphore &waitSemaphore);
@@ -70,6 +71,13 @@ class RendererVk : angle::NonCopyable
 
     Serial getCurrentQueueSerial() const;
 
+    template <typename T>
+    void enqueueGarbage(Serial serial, T &&object)
+    {
+        mGarbage.emplace_back(std::unique_ptr<vk::GarbageObject<T>>(
+            new vk::GarbageObject<T>(serial, std::move(object))));
+    }
+
   private:
     void ensureCapsInitialized() const;
     void generateCaps(gl::Caps *outCaps,
@@ -103,6 +111,7 @@ class RendererVk : angle::NonCopyable
     Serial mCurrentQueueSerial;
     Serial mLastCompletedQueueSerial;
     std::vector<vk::FenceAndCommandBuffer> mInFlightCommands;
+    std::vector<std::unique_ptr<vk::IGarbageObject>> mGarbage;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/renderervk_utils.cpp

@@ -664,6 +664,14 @@ StagingImage::StagingImage() : mSize(0)
 {
 }
 
+StagingImage::StagingImage(StagingImage &&other)
+    : mImage(std::move(other.mImage)),
+      mDeviceMemory(std::move(other.mDeviceMemory)),
+      mSize(other.mSize)
+{
+    other.mSize = 0;
+}
+
 void StagingImage::destroy(VkDevice device)
 {
     mImage.destroy(device);

src/libANGLE/renderer/vulkan/renderervk_utils.h

@@ -342,6 +342,7 @@ class StagingImage final : angle::NonCopyable
 {
   public:
     StagingImage();
+    StagingImage(StagingImage &&other);
     void destroy(VkDevice device);
     void retain(VkDevice device, StagingImage &&other);
 
@@ -441,6 +442,34 @@ class FenceAndCommandBuffer final : angle::NonCopyable
     CommandBuffer mCommandBuffer;
 };
 
+class IGarbageObject : angle::NonCopyable
+{
+  public:
+    virtual bool destroyIfComplete(VkDevice device, Serial completedSerial) = 0;
+};
+
+template <typename T>
+class GarbageObject final : public IGarbageObject
+{
+  public:
+    GarbageObject(Serial serial, T &&object) : mSerial(serial), mObject(std::move(object)) {}
+
+    bool destroyIfComplete(VkDevice device, Serial completedSerial) override
+    {
+        if (completedSerial >= mSerial)
+        {
+            mObject.destroy(device);
+            return true;
+        }
+
+        return false;
+    }
+
+  private:
+    Serial mSerial;
+    T mObject;
+};
+
 }  // namespace vk
 
 Optional<uint32_t> FindMemoryType(const VkPhysicalDeviceMemoryProperties &memoryProps,