Vulkan: Initial command queueing implementation.

src/libANGLE/renderer/vulkan/BufferVk.h

@@ -16,7 +16,7 @@
 namespace rx
 {
 
-class BufferVk : public BufferImpl
+class BufferVk : public BufferImpl, public ResourceVk
 {
   public:
     BufferVk(const gl::BufferState &state);

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -72,6 +72,7 @@ gl::Error ContextVk::drawArrays(GLenum mode, GLint first, GLsizei count)
     const auto &programVk = GetImplAs<ProgramVk>(programGL);
     const auto *drawFBO   = state.getDrawFramebuffer();
     FramebufferVk *vkFBO  = GetImplAs<FramebufferVk>(drawFBO);
+    Serial queueSerial    = mRenderer->getCurrentQueueSerial();
 
     // { vertex, fragment }
     VkPipelineShaderStageCreateInfo shaderStages[2];
@@ -129,6 +130,8 @@ gl::Error ContextVk::drawArrays(GLenum mode, GLint first, GLsizei count)
             BufferVk *bufferVk = GetImplAs<BufferVk>(bufferGL);
             vertexHandles.push_back(bufferVk->getVkBuffer().getHandle());
             vertexOffsets.push_back(0);
+
+            bufferVk->setQueueSerial(queueSerial);
         }
         else
         {
@@ -269,7 +272,7 @@ gl::Error ContextVk::drawArrays(GLenum mode, GLint first, GLsizei count)
     mCurrentPipeline.retain(device, std::move(newPipeline));
 
     vk::CommandBuffer *commandBuffer = mRenderer->getCommandBuffer();
-    ANGLE_TRY(vkFBO->beginRenderPass(device, commandBuffer, state));
+    ANGLE_TRY(vkFBO->beginRenderPass(device, commandBuffer, queueSerial, state));
 
     commandBuffer->bindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, mCurrentPipeline);
     commandBuffer->bindVertexBuffers(0, vertexHandles, vertexOffsets);

src/libANGLE/renderer/vulkan/FramebufferVk.cpp

@@ -543,8 +543,28 @@ gl::Error FramebufferVk::getSamplePosition(size_t index, GLfloat *xy) const
 
 gl::Error FramebufferVk::beginRenderPass(VkDevice device,
                                          vk::CommandBuffer *commandBuffer,
+                                         Serial queueSerial,
                                          const gl::State &glState)
 {
+    // TODO(jmadill): Cache render targets.
+    for (const auto &colorAttachment : mState.getColorAttachments())
+    {
+        if (colorAttachment.isAttached())
+        {
+            RenderTargetVk *renderTarget = nullptr;
+            ANGLE_TRY(colorAttachment.getRenderTarget<RenderTargetVk>(&renderTarget));
+            renderTarget->resource->setQueueSerial(queueSerial);
+        }
+    }
+
+    const auto *depthStencilAttachment = mState.getDepthStencilAttachment();
+    if (depthStencilAttachment && depthStencilAttachment->isAttached())
+    {
+        RenderTargetVk *renderTarget = nullptr;
+        ANGLE_TRY(depthStencilAttachment->getRenderTarget<RenderTargetVk>(&renderTarget));
+        renderTarget->resource->setQueueSerial(queueSerial);
+    }
+
     vk::Framebuffer *framebuffer = nullptr;
     ANGLE_TRY_RESULT(getFramebuffer(device), framebuffer);
     ASSERT(framebuffer && framebuffer->valid());
@@ -575,6 +595,13 @@ gl::Error FramebufferVk::beginRenderPass(VkDevice device,
     ANGLE_TRY(commandBuffer->begin(device));
     commandBuffer->beginRenderPass(*renderPass, *framebuffer, glState.getViewport(),
                                    attachmentClearValues);
+
+    setQueueSerial(queueSerial);
+    if (mBackbuffer)
+    {
+        mBackbuffer->setQueueSerial(queueSerial);
+    }
+
     return gl::NoError();
 }
 

src/libANGLE/renderer/vulkan/FramebufferVk.h

@@ -18,7 +18,7 @@ namespace rx
 class RenderTargetVk;
 class WindowSurfaceVk;
 
-class FramebufferVk : public FramebufferImpl
+class FramebufferVk : public FramebufferImpl, public ResourceVk
 {
   public:
     // Factory methods so we don't have to use constructors with overloads.
@@ -81,6 +81,7 @@ class FramebufferVk : public FramebufferImpl
 
     gl::Error beginRenderPass(VkDevice device,
                               vk::CommandBuffer *commandBuffer,
+                              Serial queueSerial,
                               const gl::State &glState);
 
     gl::ErrorOrResult<vk::RenderPass *> getRenderPass(VkDevice device);

src/libANGLE/renderer/vulkan/RenderTargetVk.cpp

@@ -13,7 +13,12 @@ namespace rx
 {
 
 RenderTargetVk::RenderTargetVk()
-    : format(nullptr), image(nullptr), imageView(nullptr), extents(), samples(VK_SAMPLE_COUNT_1_BIT)
+    : format(nullptr),
+      image(nullptr),
+      imageView(nullptr),
+      extents(),
+      samples(VK_SAMPLE_COUNT_1_BIT),
+      resource(nullptr)
 {
 }
 

src/libANGLE/renderer/vulkan/RenderTargetVk.h

@@ -16,6 +16,7 @@
 
 namespace rx
 {
+class ResourceVk;
 
 namespace vk
 {
@@ -37,6 +38,7 @@ class RenderTargetVk final : public FramebufferAttachmentRenderTarget
     vk::ImageView *imageView;
     gl::Extents extents;
     VkSampleCountFlagBits samples;
+    ResourceVk *resource;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -94,12 +94,25 @@ RendererVk::RendererVk()
       mCurrentQueueFamilyIndex(std::numeric_limits<uint32_t>::max()),
       mDevice(VK_NULL_HANDLE),
       mHostVisibleMemoryIndex(std::numeric_limits<uint32_t>::max()),
-      mGlslangWrapper(nullptr)
+      mGlslangWrapper(nullptr),
+      mCurrentQueueSerial(),
+      mLastCompletedQueueSerial(),
+      mInFlightCommands()
 {
+    ++mCurrentQueueSerial;
 }
 
 RendererVk::~RendererVk()
 {
+    if (!mInFlightCommands.empty())
+    {
+        vk::Error error = finish();
+        if (error.isError())
+        {
+            ERR() << "Error during VK shutdown: " << error;
+        }
+    }
+
     if (mGlslangWrapper)
     {
         GlslangWrapper::ReleaseReference();
@@ -584,10 +597,10 @@ vk::Error RendererVk::submitAndFinishCommandBuffer(const vk::CommandBuffer &comm
     submitInfo.pSignalSemaphores    = nullptr;
 
     // TODO(jmadill): Investigate how to properly submit command buffers.
-    ANGLE_VK_TRY(vkQueueSubmit(mQueue, 1, &submitInfo, VK_NULL_HANDLE));
+    ANGLE_TRY(submit(submitInfo));
 
     // Wait indefinitely for the queue to finish.
-    ANGLE_VK_TRY(vkQueueWaitIdle(mQueue));
+    ANGLE_TRY(finish());
 
     return vk::NoError();
 }
@@ -611,11 +624,71 @@ vk::Error RendererVk::waitThenFinishCommandBuffer(const vk::CommandBuffer &comma
     submitInfo.pSignalSemaphores    = nullptr;
 
     // TODO(jmadill): Investigate how to properly queue command buffer work.
-    ANGLE_VK_TRY(vkQueueSubmit(mQueue, 1, &submitInfo, VK_NULL_HANDLE));
+    ANGLE_TRY(submit(submitInfo));
 
     // Wait indefinitely for the queue to finish.
+    ANGLE_TRY(finish());
+
+    return vk::NoError();
+}
+
+vk::Error RendererVk::finish()
+{
+    ASSERT(mQueue != VK_NULL_HANDLE);
+    checkInFlightCommands();
     ANGLE_VK_TRY(vkQueueWaitIdle(mQueue));
+    return vk::NoError();
+}
+
+vk::Error RendererVk::checkInFlightCommands()
+{
+    // Check if any in-flight command buffers are finished.
+    for (size_t index = 0; index < mInFlightCommands.size();)
+    {
+        auto *inFlightCommand = &mInFlightCommands[index];
+
+        bool done = false;
+        ANGLE_TRY_RESULT(inFlightCommand->finished(mDevice), done);
+        if (done)
+        {
+            ASSERT(inFlightCommand->queueSerial() > mLastCompletedQueueSerial);
+            mLastCompletedQueueSerial = inFlightCommand->queueSerial();
+            inFlightCommand->destroy(mDevice);
+            mInFlightCommands.erase(mInFlightCommands.begin() + index);
+        }
+        else
+        {
+            ++index;
+        }
+    }
+
+    return vk::NoError();
+}
+
+vk::Error RendererVk::submit(const VkSubmitInfo &submitInfo)
+{
+    checkInFlightCommands();
+
+    // Start a Fence to record when this command buffer finishes.
+    VkFenceCreateInfo fenceInfo;
+    fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+    fenceInfo.pNext = nullptr;
+    fenceInfo.flags = 0;
+
+    vk::Fence fence;
+    ANGLE_TRY(fence.init(mDevice, fenceInfo));
+
+    ANGLE_VK_TRY(vkQueueSubmit(mQueue, 1, &submitInfo, fence.getHandle()));
 
+    // Store this command buffer in the in-flight list.
+    mInFlightCommands.emplace_back(vk::FenceAndCommandBuffer(mCurrentQueueSerial, std::move(fence),
+                                                             std::move(mCommandBuffer)));
+
+    // Sanity check.
+    ASSERT(mInFlightCommands.size() < 1000u);
+
+    // Increment the command buffer serial. If this fails, we need to restart ANGLE.
+    ANGLE_VK_CHECK(++mCurrentQueueSerial, VK_ERROR_OUT_OF_HOST_MEMORY);
     return vk::NoError();
 }
 
@@ -637,4 +710,9 @@ GlslangWrapper *RendererVk::getGlslangWrapper()
     return mGlslangWrapper;
 }
 
+Serial RendererVk::getCurrentQueueSerial() const
+{
+    return mCurrentQueueSerial;
+}
+
 }  // namespace rx

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -54,6 +54,7 @@ class RendererVk : angle::NonCopyable
     vk::Error submitAndFinishCommandBuffer(const vk::CommandBuffer &commandBuffer);
     vk::Error waitThenFinishCommandBuffer(const vk::CommandBuffer &commandBuffer,
                                           const vk::Semaphore &waitSemaphore);
+    vk::Error finish();
 
     const gl::Caps &getNativeCaps() const;
     const gl::TextureCapsMap &getNativeTextureCaps() const;
@@ -67,12 +68,16 @@ class RendererVk : angle::NonCopyable
 
     GlslangWrapper *getGlslangWrapper();
 
+    Serial getCurrentQueueSerial() const;
+
   private:
     void ensureCapsInitialized() const;
     void generateCaps(gl::Caps *outCaps,
                       gl::TextureCapsMap *outTextureCaps,
                       gl::Extensions *outExtensions,
                       gl::Limitations *outLimitations) const;
+    vk::Error submit(const VkSubmitInfo &submitInfo);
+    vk::Error checkInFlightCommands();
 
     mutable bool mCapsInitialized;
     mutable gl::Caps mNativeCaps;
@@ -95,6 +100,9 @@ class RendererVk : angle::NonCopyable
     vk::CommandBuffer mCommandBuffer;
     uint32_t mHostVisibleMemoryIndex;
     GlslangWrapper *mGlslangWrapper;
+    Serial mCurrentQueueSerial;
+    Serial mLastCompletedQueueSerial;
+    std::vector<vk::FenceAndCommandBuffer> mInFlightCommands;
 };
 
 }  // namespace rx

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -150,6 +150,7 @@ WindowSurfaceVk::WindowSurfaceVk(const egl::SurfaceState &surfaceState,
     mRenderTarget.extents.width  = static_cast<GLint>(width);
     mRenderTarget.extents.height = static_cast<GLint>(height);
     mRenderTarget.extents.depth  = 1;
+    mRenderTarget.resource       = this;
 }
 
 WindowSurfaceVk::~WindowSurfaceVk()

src/libANGLE/renderer/vulkan/SurfaceVk.h

@@ -50,7 +50,7 @@ class OffscreenSurfaceVk : public SurfaceImpl
     EGLint mHeight;
 };
 
-class WindowSurfaceVk : public SurfaceImpl
+class WindowSurfaceVk : public SurfaceImpl, public ResourceVk
 {
   public:
     WindowSurfaceVk(const egl::SurfaceState &surfaceState,
@@ -83,7 +83,6 @@ class WindowSurfaceVk : public SurfaceImpl
     gl::ErrorOrResult<vk::Framebuffer *> getCurrentFramebuffer(
         VkDevice device,
         const vk::RenderPass &compatibleRenderPass);
-    void onBeginRenderPass();
 
   private:
     vk::Error initializeImpl(RendererVk *renderer);

src/libANGLE/renderer/vulkan/renderervk_utils.cpp

@@ -177,7 +177,7 @@ gl::Error Error::toGL(GLenum glErrorCode) const
     }
 
     // TODO(jmadill): Set extended error code to 'vulkan internal error'.
-    const std::string &message = getExtendedMessage();
+    const std::string &message = toString();
     return gl::Error(glErrorCode, message.c_str());
 }
 
@@ -189,11 +189,11 @@ egl::Error Error::toEGL(EGLint eglErrorCode) const
     }
 
     // TODO(jmadill): Set extended error code to 'vulkan internal error'.
-    const std::string &message = getExtendedMessage();
+    const std::string &message = toString();
     return egl::Error(eglErrorCode, message.c_str());
 }
 
-std::string Error::getExtendedMessage() const
+std::string Error::toString() const
 {
     std::stringstream errorStream;
     errorStream << "Internal Vulkan error: " << VulkanResultString(mResult) << ", in " << mFile
@@ -827,6 +827,73 @@ Error PipelineLayout::init(VkDevice device, const VkPipelineLayoutCreateInfo &cr
     return NoError();
 }
 
+// Fence implementation.
+Fence::Fence()
+{
+}
+
+void Fence::destroy(VkDevice device)
+{
+    if (valid())
+    {
+        vkDestroyFence(device, mHandle, nullptr);
+        mHandle = VK_NULL_HANDLE;
+    }
+}
+
+Error Fence::init(VkDevice device, const VkFenceCreateInfo &createInfo)
+{
+    ASSERT(!valid());
+    ANGLE_VK_TRY(vkCreateFence(device, &createInfo, nullptr, &mHandle));
+    return NoError();
+}
+
+VkResult Fence::getStatus(VkDevice device) const
+{
+    return vkGetFenceStatus(device, mHandle);
+}
+
+// FenceAndCommandBuffer implementation.
+FenceAndCommandBuffer::FenceAndCommandBuffer(Serial queueSerial,
+                                             Fence &&fence,
+                                             CommandBuffer &&commandBuffer)
+    : mQueueSerial(queueSerial), mFence(std::move(fence)), mCommandBuffer(std::move(commandBuffer))
+{
+}
+
+FenceAndCommandBuffer::FenceAndCommandBuffer(FenceAndCommandBuffer &&other)
+    : mQueueSerial(std::move(other.mQueueSerial)),
+      mFence(std::move(other.mFence)),
+      mCommandBuffer(std::move(other.mCommandBuffer))
+{
+}
+
+void FenceAndCommandBuffer::destroy(VkDevice device)
+{
+    mFence.destroy(device);
+    mCommandBuffer.destroy(device);
+}
+
+vk::ErrorOrResult<bool> FenceAndCommandBuffer::finished(VkDevice device) const
+{
+    VkResult result = mFence.getStatus(device);
+    // Should this be a part of ANGLE_VK_TRY?
+    if (result == VK_NOT_READY)
+    {
+        return false;
+    }
+    ANGLE_VK_TRY(result);
+    return true;
+}
+
+FenceAndCommandBuffer &FenceAndCommandBuffer::operator=(FenceAndCommandBuffer &&other)
+{
+    std::swap(mQueueSerial, other.mQueueSerial);
+    mFence         = std::move(other.mFence);
+    mCommandBuffer = std::move(other.mCommandBuffer);
+    return *this;
+}
+
 }  // namespace vk
 
 Optional<uint32_t> FindMemoryType(const VkPhysicalDeviceMemoryProperties &memoryProps,
@@ -912,3 +979,9 @@ VkFrontFace GetFrontFace(GLenum frontFace)
 }  // namespace gl_vk
 
 }  // namespace rx
+
+std::ostream &operator<<(std::ostream &stream, const rx::vk::Error &error)
+{
+    stream << error.toString();
+    return stream;
+}

src/libANGLE/renderer/vulkan/renderervk_utils.h

@@ -39,6 +39,71 @@ enum class TextureDimension
     TEX_2D_ARRAY,
 };
 
+enum DeleteSchedule
+{
+    NOW,
+    LATER,
+};
+
+// A serial supports a few operations - comparison, increment, and assignment.
+// TODO(jmadill): Verify it's not easy to overflow the queue serial.
+class Serial final
+{
+  public:
+    Serial() : mValue(0) {}
+    Serial(const Serial &other) : mValue(other.mValue) {}
+    Serial(Serial &&other) : mValue(other.mValue) { other.mValue = 0; }
+    Serial &operator=(const Serial &other)
+    {
+        mValue = other.mValue;
+        return *this;
+    }
+    bool operator>=(Serial other) const { return mValue >= other.mValue; }
+    bool operator>(Serial other) const { return mValue > other.mValue; }
+
+    // This function fails if we're at the limits of our counting.
+    bool operator++()
+    {
+        if (mValue == std::numeric_limits<uint32_t>::max())
+            return false;
+        mValue++;
+        return true;
+    }
+
+  private:
+    uint32_t mValue;
+};
+
+// This is a small helper mixin for any GL object used in Vk command buffers. It records a serial
+// at command submission times indicating it's order in the queue. We will use Fences to detect
+// when commands are finished, and then handle lifetime management for the resources.
+// Note that we use a queue order serial instead of a command buffer id serial since a queue can
+// submit multiple command buffers in one API call.
+class ResourceVk
+{
+  public:
+    void setQueueSerial(Serial queueSerial)
+    {
+        ASSERT(queueSerial >= mStoredQueueSerial);
+        mStoredQueueSerial = queueSerial;
+    }
+
+    DeleteSchedule getDeleteSchedule(Serial lastCompletedQueueSerial)
+    {
+        if (lastCompletedQueueSerial >= mStoredQueueSerial)
+        {
+            return DeleteSchedule::NOW;
+        }
+        else
+        {
+            return DeleteSchedule::LATER;
+        }
+    }
+
+  private:
+    Serial mStoredQueueSerial;
+};
+
 namespace vk
 {
 class DeviceMemory;
@@ -70,9 +135,9 @@ class Error final
 
     bool isError() const;
 
-  private:
-    std::string getExtendedMessage() const;
+    std::string toString() const;
 
+  private:
     VkResult mResult;
     const char *mFile;
     unsigned int mLine;
@@ -99,12 +164,19 @@ class WrappedObject : angle::NonCopyable
     WrappedObject(HandleT handle) : mHandle(handle) {}
     ~WrappedObject() { ASSERT(!valid()); }
 
-    // Only works to initialize empty objects, since we don't have the device handle.
     WrappedObject(WrappedObject &&other) : mHandle(other.mHandle)
     {
         other.mHandle = VK_NULL_HANDLE;
     }
 
+    // Only works to initialize empty objects, since we don't have the device handle.
+    WrappedObject &operator=(WrappedObject &&other)
+    {
+        ASSERT(!valid());
+        std::swap(mHandle, other.mHandle);
+        return *this;
+    }
+
     void retain(VkDevice device, DerivedT &&other)
     {
         if (valid())
@@ -134,6 +206,7 @@ class CommandBuffer final : public WrappedObject<CommandBuffer, VkCommandBuffer>
     CommandBuffer();
 
     void destroy(VkDevice device);
+    using WrappedObject::operator=;
 
     void setCommandPool(CommandPool *commandPool);
     Error begin(VkDevice device);
@@ -338,6 +411,36 @@ class PipelineLayout final : public WrappedObject<PipelineLayout, VkPipelineLayo
     Error init(VkDevice device, const VkPipelineLayoutCreateInfo &createInfo);
 };
 
+class Fence final : public WrappedObject<Fence, VkFence>
+{
+  public:
+    Fence();
+    void destroy(VkDevice fence);
+    using WrappedObject::retain;
+    using WrappedObject::operator=;
+
+    Error init(VkDevice device, const VkFenceCreateInfo &createInfo);
+    VkResult getStatus(VkDevice device) const;
+};
+
+class FenceAndCommandBuffer final : angle::NonCopyable
+{
+  public:
+    FenceAndCommandBuffer(Serial queueSerial, Fence &&fence, CommandBuffer &&commandBuffer);
+    FenceAndCommandBuffer(FenceAndCommandBuffer &&other);
+    FenceAndCommandBuffer &operator=(FenceAndCommandBuffer &&other);
+
+    void destroy(VkDevice device);
+    vk::ErrorOrResult<bool> finished(VkDevice device) const;
+
+    Serial queueSerial() const { return mQueueSerial; }
+
+  private:
+    Serial mQueueSerial;
+    Fence mFence;
+    CommandBuffer mCommandBuffer;
+};
+
 }  // namespace vk
 
 Optional<uint32_t> FindMemoryType(const VkPhysicalDeviceMemoryProperties &memoryProps,
@@ -365,4 +468,6 @@ VkFrontFace GetFrontFace(GLenum frontFace);
 
 #define ANGLE_VK_CHECK(test, error) ANGLE_VK_TRY(test ? VK_SUCCESS : error)
 
+std::ostream &operator<<(std::ostream &stream, const rx::vk::Error &error);
+
 #endif  // LIBANGLE_RENDERER_VULKAN_RENDERERVK_UTILS_H_

src/tests/gl_tests/SimpleOperationTest.cpp

@@ -278,6 +278,7 @@ TEST_P(SimpleOperationTest, DrawQuadAndSwap)
     {
         drawQuad(program.get(), "position", 0.5f, 1.0f, true);
         EXPECT_GL_NO_ERROR();
+        EXPECT_PIXEL_COLOR_EQ(0, 0, GLColor::green);
         swapBuffers();
     }