Vulkan: Implement glFlush

src/common/FixedVector.h

@@ -291,7 +291,17 @@ void FixedVector<T, N, Storage>::swap(FixedVector<T, N, Storage> &other)
 template <class T, size_t N, class Storage>
 void FixedVector<T, N, Storage>::resize(size_type count)
 {
-    resize(count, value_type());
+    ASSERT(count <= N);
+    while (mSize > count)
+    {
+        mSize--;
+        mStorage[mSize] = value_type();
+    }
+    while (mSize < count)
+    {
+        mStorage[mSize] = value_type();
+        mSize++;
+    }
 }
 
 template <class T, size_t N, class Storage>
@@ -301,7 +311,7 @@ void FixedVector<T, N, Storage>::resize(size_type count, const value_type &value
     while (mSize > count)
     {
         mSize--;
-        mStorage[mSize] = T();
+        mStorage[mSize] = value_type();
     }
     while (mSize < count)
     {

src/libANGLE/renderer/vulkan/CommandGraph.cpp

@@ -571,6 +571,10 @@ angle::Result CommandGraph::submitCommands(Context *context,
                                            CommandPool *commandPool,
                                            CommandBuffer *primaryCommandBufferOut)
 {
+    // There is no point in submitting an empty command buffer, so make sure not to call this
+    // function if there's nothing to do.
+    ASSERT(!mNodes.empty());
+
     size_t previousBarrierIndex       = 0;
     CommandGraphNode *previousBarrier = getLastBarrierNode(&previousBarrierIndex);
 
@@ -582,6 +586,8 @@ angle::Result CommandGraph::submitCommands(Context *context,
             previousBarrier, &mNodes[previousBarrierIndex + 1], afterNodesCount);
     }
 
+    mLastBarrierIndex = kInvalidNodeIndex;
+
     VkCommandBufferAllocateInfo primaryInfo = {};
     primaryInfo.sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
     primaryInfo.commandPool        = commandPool->getHandle();
@@ -590,11 +596,6 @@ angle::Result CommandGraph::submitCommands(Context *context,
 
     ANGLE_TRY(primaryCommandBufferOut->init(context, primaryInfo));
 
-    if (mNodes.empty())
-    {
-        return angle::Result::Continue();
-    }
-
     if (mEnableGraphDiagnostics)
     {
         dumpGraphDotFile(std::cout);
@@ -650,7 +651,6 @@ angle::Result CommandGraph::submitCommands(Context *context,
         delete node;
     }
     mNodes.clear();
-    mLastBarrierIndex = kInvalidNodeIndex;
 
     return angle::Result::Continue();
 }

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -220,13 +220,7 @@ gl::Error ContextVk::initialize()
 
 gl::Error ContextVk::flush(const gl::Context *context)
 {
-    // TODO(jmadill): Multiple flushes will need to insert semaphores. http://anglebug.com/2504
-
-    // dEQP tests rely on having no errors thrown at the end of the test and they always call
-    // flush at the end of the their tests. Just returning NoError until we implement flush
-    // allow us to work on enabling many tests in the meantime.
-    WARN() << "Flush is unimplemented. http://anglebug.com/2504";
-    return gl::NoError();
+    return mRenderer->flush(this);
 }
 
 gl::Error ContextVk::finish(const gl::Context *context)

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -326,6 +326,7 @@ void RendererVk::onDestroy(vk::Context *context)
     mRenderPassCache.destroy(mDevice);
     mPipelineCache.destroy(mDevice);
     mPipelineCacheVk.destroy(mDevice);
+    mSubmitSemaphorePool.destroy(mDevice);
     mShaderLibrary.destroy(mDevice);
 
     GlslangWrapper::Release();
@@ -615,9 +616,12 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF
 
     ANGLE_TRY(mCommandPool.init(displayVk, commandPoolInfo));
 
-    // Initialize the vulkan pipeline cache
+    // Initialize the vulkan pipeline cache.
     ANGLE_TRY(initPipelineCacheVk(displayVk));
 
+    // Initialize the submission semaphore pool.
+    ANGLE_TRY(mSubmitSemaphorePool.init(displayVk, vk::kDefaultSemaphorePoolSize));
+
     return angle::Result::Continue();
 }
 
@@ -771,6 +775,27 @@ void RendererVk::ensureCapsInitialized() const
     }
 }
 
+void RendererVk::getSubmitWaitSemaphores(
+    vk::Context *context,
+    angle::FixedVector<VkSemaphore, kMaxWaitSemaphores> *waitSemaphores)
+{
+    if (mSubmitLastSignaledSemaphore.getSemaphore())
+    {
+        waitSemaphores->push_back(mSubmitLastSignaledSemaphore.getSemaphore()->getHandle());
+
+        // Return the semaphore to the pool (which will remain valid and unused until the
+        // queue it's about to be waited on has finished execution).
+        mSubmitSemaphorePool.freeSemaphore(context, &mSubmitLastSignaledSemaphore);
+    }
+
+    for (vk::SemaphoreHelper &semaphore : mSubmitWaitSemaphores)
+    {
+        waitSemaphores->push_back(semaphore.getSemaphore()->getHandle());
+        mSubmitSemaphorePool.freeSemaphore(context, &semaphore);
+    }
+    mSubmitWaitSemaphores.clear();
+}
+
 const gl::Caps &RendererVk::getNativeCaps() const
 {
     ensureCapsInitialized();
@@ -814,11 +839,18 @@ angle::Result RendererVk::finish(vk::Context *context)
         vk::Scoped<vk::CommandBuffer> commandBatch(mDevice);
         ANGLE_TRY(flushCommandGraph(context, &commandBatch.get()));
 
+        VkPipelineStageFlags waitStageMask[2] = {
+            VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
+        };
+
+        angle::FixedVector<VkSemaphore, kMaxWaitSemaphores> waitSemaphores;
+        getSubmitWaitSemaphores(context, &waitSemaphores);
+
         VkSubmitInfo submitInfo         = {};
         submitInfo.sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-        submitInfo.waitSemaphoreCount   = 0;
-        submitInfo.pWaitSemaphores      = nullptr;
-        submitInfo.pWaitDstStageMask    = nullptr;
+        submitInfo.waitSemaphoreCount   = static_cast<uint32_t>(waitSemaphores.size());
+        submitInfo.pWaitSemaphores      = waitSemaphores.data();
+        submitInfo.pWaitDstStageMask    = waitStageMask;
         submitInfo.commandBufferCount   = 1;
         submitInfo.pCommandBuffers      = commandBatch.get().ptr();
         submitInfo.signalSemaphoreCount = 0;
@@ -967,26 +999,39 @@ angle::Result RendererVk::flushCommandGraph(vk::Context *context, vk::CommandBuf
                                         &mCommandPool, commandBatch);
 }
 
-angle::Result RendererVk::flush(vk::Context *context,
-                                const vk::Semaphore &waitSemaphore,
-                                const vk::Semaphore &signalSemaphore)
+angle::Result RendererVk::flush(vk::Context *context)
 {
+    if (mCommandGraph.empty())
+    {
+        return angle::Result::Continue();
+    }
+
     vk::Scoped<vk::CommandBuffer> commandBatch(mDevice);
     ANGLE_TRY(flushCommandGraph(context, &commandBatch.get()));
 
-    VkPipelineStageFlags waitStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
+    VkPipelineStageFlags waitStageMask[2] = {
+        VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
+    };
+
+    angle::FixedVector<VkSemaphore, kMaxWaitSemaphores> waitSemaphores;
+    getSubmitWaitSemaphores(context, &waitSemaphores);
+
+    // On every flush, create a semaphore to be signaled.  On the next submission, this semaphore
+    // will be waited on.
+    ANGLE_TRY(mSubmitSemaphorePool.allocateSemaphore(context, &mSubmitLastSignaledSemaphore));
 
     VkSubmitInfo submitInfo         = {};
     submitInfo.sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-    submitInfo.waitSemaphoreCount   = 1;
-    submitInfo.pWaitSemaphores      = waitSemaphore.ptr();
-    submitInfo.pWaitDstStageMask    = &waitStageMask;
+    submitInfo.waitSemaphoreCount   = static_cast<uint32_t>(waitSemaphores.size());
+    submitInfo.pWaitSemaphores      = waitSemaphores.data();
+    submitInfo.pWaitDstStageMask    = waitStageMask;
     submitInfo.commandBufferCount   = 1;
     submitInfo.pCommandBuffers      = commandBatch.get().ptr();
     submitInfo.signalSemaphoreCount = 1;
-    submitInfo.pSignalSemaphores    = signalSemaphore.ptr();
+    submitInfo.pSignalSemaphores    = mSubmitLastSignaledSemaphore.getSemaphore()->ptr();
 
     ANGLE_TRY(submitFrame(context, submitInfo, commandBatch.release()));
+
     return angle::Result::Continue();
 }
 
@@ -1079,6 +1124,32 @@ angle::Result RendererVk::syncPipelineCacheVk(DisplayVk *displayVk)
     return angle::Result::Continue();
 }
 
+angle::Result RendererVk::allocateSubmitWaitSemaphore(vk::Context *context,
+                                                      const vk::Semaphore **outSemaphore)
+{
+    ASSERT(mSubmitWaitSemaphores.size() < mSubmitWaitSemaphores.max_size());
+
+    vk::SemaphoreHelper semaphore;
+    ANGLE_TRY(mSubmitSemaphorePool.allocateSemaphore(context, &semaphore));
+
+    mSubmitWaitSemaphores.push_back(std::move(semaphore));
+    *outSemaphore = mSubmitWaitSemaphores.back().getSemaphore();
+
+    return angle::Result::Continue();
+}
+
+const vk::Semaphore *RendererVk::getSubmitLastSignaledSemaphore(vk::Context *context)
+{
+    const vk::Semaphore *semaphore = mSubmitLastSignaledSemaphore.getSemaphore();
+
+    // Return the semaphore to the pool (which will remain valid and unused until the
+    // queue it's about to be waited on has finished execution).  The caller is about
+    // to wait on it.
+    mSubmitSemaphorePool.freeSemaphore(context, &mSubmitLastSignaledSemaphore);
+
+    return semaphore;
+}
+
 vk::ShaderLibrary *RendererVk::getShaderLibrary()
 {
     return &mShaderLibrary;

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -19,6 +19,7 @@
 #include "libANGLE/renderer/vulkan/CommandGraph.h"
 #include "libANGLE/renderer/vulkan/FeaturesVk.h"
 #include "libANGLE/renderer/vulkan/vk_format_utils.h"
+#include "libANGLE/renderer/vulkan/vk_helpers.h"
 #include "libANGLE/renderer/vulkan/vk_internal_shaders.h"
 
 namespace egl
@@ -72,9 +73,7 @@ class RendererVk : angle::NonCopyable
                                                uint32_t *presentQueueOut);
 
     angle::Result finish(vk::Context *context);
-    angle::Result flush(vk::Context *context,
-                        const vk::Semaphore &waitSemaphore,
-                        const vk::Semaphore &signalSemaphore);
+    angle::Result flush(vk::Context *context);
 
     const vk::CommandPool &getCommandPool() const;
 
@@ -145,6 +144,16 @@ class RendererVk : angle::NonCopyable
 
     angle::Result syncPipelineCacheVk(DisplayVk *displayVk);
 
+    vk::DynamicSemaphorePool *getDynamicSemaphorePool() { return &mSubmitSemaphorePool; }
+
+    // Request a semaphore, that is expected to be signaled externally.  The next submission will
+    // wait on it.
+    angle::Result allocateSubmitWaitSemaphore(vk::Context *context,
+                                              const vk::Semaphore **outSemaphore);
+    // Get the last signaled semaphore to wait on externally.  The semaphore will not be waited on
+    // by next submission.
+    const vk::Semaphore *getSubmitLastSignaledSemaphore(vk::Context *context);
+
     // This should only be called from ResourceVk.
     // TODO(jmadill): Keep in ContextVk to enable threaded rendering.
     vk::CommandGraph *getCommandGraph();
@@ -156,8 +165,18 @@ class RendererVk : angle::NonCopyable
     const FeaturesVk &getFeatures() const { return mFeatures; }
 
   private:
+    // Number of semaphores for external entities to renderer to issue a wait, such as surface's
+    // image acquire.
+    static constexpr size_t kMaxExternalSemaphores = 8;
+    // Total possible number of semaphores a submission can wait on.  +1 is for the semaphore
+    // signaled in the last submission.
+    static constexpr size_t kMaxWaitSemaphores = kMaxExternalSemaphores + 1;
+
     angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
+    void getSubmitWaitSemaphores(
+        vk::Context *context,
+        angle::FixedVector<VkSemaphore, kMaxWaitSemaphores> *waitSemaphores);
     angle::Result submitFrame(vk::Context *context,
                               const VkSubmitInfo &submitInfo,
                               vk::CommandBuffer &&commandBuffer);
@@ -219,6 +238,26 @@ class RendererVk : angle::NonCopyable
     egl::BlobCache::Key mPipelineCacheVkBlobKey;
     uint32_t mPipelineCacheVkUpdateTimeout;
 
+    // mSubmitWaitSemaphores is a list of specifically requested semaphores to be waited on before a
+    // command buffer submission, for example, semaphores signaled by vkAcquireNextImageKHR.
+    // After first use, the list is automatically cleared.  This is a vector to support concurrent
+    // rendering to multiple surfaces.
+    //
+    // Note that with multiple contexts present, this may result in a context waiting on image
+    // acquisition even if it doesn't render to that surface.  If CommandGraphs are separated by
+    // context or share group for example, this could be moved to the one that actually uses the
+    // image.
+    angle::FixedVector<vk::SemaphoreHelper, kMaxExternalSemaphores> mSubmitWaitSemaphores;
+    // mSubmitLastSignaledSemaphore shows which semaphore was last signaled by submission.  This can
+    // be set to nullptr if retrieved to be waited on outside RendererVk, such
+    // as by the surface before presentation.  Each submission waits on the
+    // previously signaled semaphore (as well as any in mSubmitWaitSemaphores)
+    // and allocates a new semaphore to signal.
+    vk::SemaphoreHelper mSubmitLastSignaledSemaphore;
+
+    // A pool of semaphores used to support the aforementioned mid-frame submissions.
+    vk::DynamicSemaphorePool mSubmitSemaphorePool;
+
     // See CommandGraph.h for a desription of the Command Graph.
     vk::CommandGraph mCommandGraph;
 

src/libANGLE/renderer/vulkan/SurfaceVk.cpp

@@ -248,9 +248,7 @@ WindowSurfaceVk::SwapchainImage::~SwapchainImage() = default;
 WindowSurfaceVk::SwapchainImage::SwapchainImage(SwapchainImage &&other)
     : image(std::move(other.image)),
       imageView(std::move(other.imageView)),
-      framebuffer(std::move(other.framebuffer)),
-      imageAcquiredSemaphore(std::move(other.imageAcquiredSemaphore)),
-      commandsCompleteSemaphore(std::move(other.commandsCompleteSemaphore))
+      framebuffer(std::move(other.framebuffer))
 {
 }
 
@@ -285,8 +283,6 @@ void WindowSurfaceVk::destroy(const egl::Display *display)
     // We might not need to flush the pipe here.
     (void)renderer->finish(displayVk);
 
-    mAcquireNextImageSemaphore.destroy(device);
-
     mDepthStencilImage.release(renderer);
     mDepthStencilImageView.destroy(device);
 
@@ -297,8 +293,6 @@ void WindowSurfaceVk::destroy(const egl::Display *display)
         swapchainImage.image.destroy(device);
         swapchainImage.imageView.destroy(device);
         swapchainImage.framebuffer.destroy(device);
-        swapchainImage.imageAcquiredSemaphore.destroy(device);
-        swapchainImage.commandsCompleteSemaphore.destroy(device);
     }
 
     if (mSwapchain)
@@ -480,8 +474,6 @@ angle::Result WindowSurfaceVk::initializeImpl(DisplayVk *displayVk)
 
     mSwapchainImages.resize(imageCount);
 
-    ANGLE_TRY(mAcquireNextImageSemaphore.init(displayVk));
-
     for (uint32_t imageIndex = 0; imageIndex < imageCount; ++imageIndex)
     {
         SwapchainImage &member = mSwapchainImages[imageIndex];
@@ -496,9 +488,6 @@ angle::Result WindowSurfaceVk::initializeImpl(DisplayVk *displayVk)
 
         // Set transfer dest layout, and clear the image to black.
         member.image.clearColor(transparentBlack, 0, 1, commandBuffer);
-
-        ANGLE_TRY(member.imageAcquiredSemaphore.init(displayVk));
-        ANGLE_TRY(member.commandsCompleteSemaphore.init(displayVk));
     }
 
     // Get the first available swapchain iamge.
@@ -570,13 +559,17 @@ angle::Result WindowSurfaceVk::swapImpl(DisplayVk *displayVk)
                                        VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
                                        VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, swapCommands);
 
-    ANGLE_TRY(
-        renderer->flush(displayVk, image.imageAcquiredSemaphore, image.commandsCompleteSemaphore));
+    ANGLE_TRY(renderer->flush(displayVk));
+
+    // Ask the renderer what semaphore it signalled in the last flush.
+    const vk::Semaphore *commandsCompleteSemaphore =
+        renderer->getSubmitLastSignaledSemaphore(displayVk);
 
     VkPresentInfoKHR presentInfo   = {};
     presentInfo.sType              = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
-    presentInfo.waitSemaphoreCount = 1;
-    presentInfo.pWaitSemaphores    = image.commandsCompleteSemaphore.ptr();
+    presentInfo.waitSemaphoreCount = commandsCompleteSemaphore ? 1 : 0;
+    presentInfo.pWaitSemaphores =
+        commandsCompleteSemaphore ? commandsCompleteSemaphore->ptr() : nullptr;
     presentInfo.swapchainCount     = 1;
     presentInfo.pSwapchains        = &mSwapchain;
     presentInfo.pImageIndices      = &mCurrentSwapchainImageIndex;
@@ -595,16 +588,17 @@ angle::Result WindowSurfaceVk::swapImpl(DisplayVk *displayVk)
 angle::Result WindowSurfaceVk::nextSwapchainImage(DisplayVk *displayVk)
 {
     VkDevice device = displayVk->getDevice();
+    RendererVk *renderer = displayVk->getRenderer();
+
+    const vk::Semaphore *acquireNextImageSemaphore = nullptr;
+    ANGLE_TRY(renderer->allocateSubmitWaitSemaphore(displayVk, &acquireNextImageSemaphore));
 
     ANGLE_VK_TRY(displayVk, vkAcquireNextImageKHR(device, mSwapchain, UINT64_MAX,
-                                                  mAcquireNextImageSemaphore.getHandle(),
+                                                  acquireNextImageSemaphore->getHandle(),
                                                   VK_NULL_HANDLE, &mCurrentSwapchainImageIndex));
 
     SwapchainImage &image = mSwapchainImages[mCurrentSwapchainImageIndex];
 
-    // Swap the unused swapchain semaphore and the now active spare semaphore.
-    std::swap(image.imageAcquiredSemaphore, mAcquireNextImageSemaphore);
-
     // Update RenderTarget pointers.
     mColorRenderTarget.updateSwapchainImage(&image.image, &image.imageView);
 

src/libANGLE/renderer/vulkan/SurfaceVk.h

@@ -152,13 +152,6 @@ class WindowSurfaceVk : public SurfaceImpl
 
     uint32_t mCurrentSwapchainImageIndex;
 
-    // When acquiring a new image for rendering, we keep a 'spare' semaphore. We pass this extra
-    // semaphore to VkAcquireNextImage, then hand it to the next available SwapchainImage when
-    // the command completes. We then make the old semaphore in the new SwapchainImage the spare
-    // semaphore, since we know the image is no longer using it. This avoids the chicken and egg
-    // problem with needing to know the next available image index before we acquire it.
-    vk::Semaphore mAcquireNextImageSemaphore;
-
     struct SwapchainImage : angle::NonCopyable
     {
         SwapchainImage();
@@ -168,8 +161,6 @@ class WindowSurfaceVk : public SurfaceImpl
         vk::ImageHelper image;
         vk::ImageView imageView;
         vk::Framebuffer framebuffer;
-        vk::Semaphore imageAcquiredSemaphore;
-        vk::Semaphore commandsCompleteSemaphore;
     };
 
     std::vector<SwapchainImage> mSwapchainImages;

src/libANGLE/renderer/vulkan/vk_helpers.cpp

@@ -418,45 +418,112 @@ void DynamicDescriptorPool::setMaxSetsPerPoolForTesting(uint32_t maxSetsPerPool)
     mMaxSetsPerPool = maxSetsPerPool;
 }
 
-// DynamicQueryPool implementation
-DynamicQueryPool::DynamicQueryPool() : mPoolSize(0), mCurrentQueryPool(0), mCurrentFreeQuery(0)
+// DynamicallyGrowingPool implementation
+template <typename Pool>
+DynamicallyGrowingPool<Pool>::DynamicallyGrowingPool()
+    : mPoolSize(0), mCurrentPool(0), mCurrentFreeEntry(0)
+{
+}
+
+template <typename Pool>
+DynamicallyGrowingPool<Pool>::~DynamicallyGrowingPool() = default;
+
+template <typename Pool>
+angle::Result DynamicallyGrowingPool<Pool>::initEntryPool(Context *context, uint32_t poolSize)
+{
+    ASSERT(mPools.empty() && mPoolStats.empty());
+    mPoolSize = poolSize;
+    return angle::Result::Continue();
+}
+
+template <typename Pool>
+void DynamicallyGrowingPool<Pool>::destroyEntryPool()
+{
+    mPools.clear();
+    mPoolStats.clear();
+}
+
+template <typename Pool>
+bool DynamicallyGrowingPool<Pool>::findFreeEntryPool(Context *context)
+{
+    Serial lastCompletedQueueSerial = context->getRenderer()->getLastCompletedQueueSerial();
+    for (size_t i = 0; i < mPools.size(); ++i)
+    {
+        if (mPoolStats[i].freedCount == mPoolSize &&
+            mPoolStats[i].serial <= lastCompletedQueueSerial)
+        {
+            mCurrentPool      = i;
+            mCurrentFreeEntry = 0;
+
+            mPoolStats[i].freedCount = 0;
+
+            return true;
+        }
+    }
+
+    return false;
+}
+
+template <typename Pool>
+angle::Result DynamicallyGrowingPool<Pool>::allocateNewEntryPool(Context *context, Pool &&pool)
+{
+    mPools.push_back(std::move(pool));
+
+    PoolStats poolStats = {0, Serial()};
+    mPoolStats.push_back(poolStats);
+
+    mCurrentPool      = mPools.size() - 1;
+    mCurrentFreeEntry = 0;
+
+    return angle::Result::Continue();
+}
+
+template <typename Pool>
+void DynamicallyGrowingPool<Pool>::onEntryFreed(Context *context, size_t poolIndex)
 {
+    ASSERT(poolIndex < mPoolStats.size() && mPoolStats[poolIndex].freedCount < mPoolSize);
+
+    // Take note of the current serial to avoid reallocating a query in the same pool
+    mPoolStats[poolIndex].serial = context->getRenderer()->getCurrentQueueSerial();
+    ++mPoolStats[poolIndex].freedCount;
 }
 
+// DynamicQueryPool implementation
+DynamicQueryPool::DynamicQueryPool() = default;
+
 DynamicQueryPool::~DynamicQueryPool() = default;
 
 angle::Result DynamicQueryPool::init(Context *context, VkQueryType type, uint32_t poolSize)
 {
-    ASSERT(mQueryPools.empty() && mQueryPoolStats.empty());
+    ANGLE_TRY(initEntryPool(context, poolSize));
 
-    mPoolSize  = poolSize;
     mQueryType = type;
     ANGLE_TRY(allocateNewPool(context));
+
     return angle::Result::Continue();
 }
 
 void DynamicQueryPool::destroy(VkDevice device)
 {
-    for (QueryPool &queryPool : mQueryPools)
+    for (QueryPool &queryPool : mPools)
     {
         queryPool.destroy(device);
     }
 
-    mQueryPools.clear();
-    mQueryPoolStats.clear();
+    destroyEntryPool();
 }
 
 angle::Result DynamicQueryPool::allocateQuery(Context *context, QueryHelper *queryOut)
 {
     ASSERT(!queryOut->getQueryPool());
 
-    if (mCurrentFreeQuery >= mPoolSize)
+    if (mCurrentFreeEntry >= mPoolSize)
     {
         // No more queries left in this pool, create another one.
         ANGLE_TRY(allocateNewPool(context));
     }
 
-    queryOut->init(this, mCurrentQueryPool, mCurrentFreeQuery++);
+    queryOut->init(this, mCurrentPool, mCurrentFreeEntry++);
 
     return angle::Result::Continue();
 }
@@ -466,32 +533,18 @@ void DynamicQueryPool::freeQuery(Context *context, QueryHelper *query)
     if (query->getQueryPool())
     {
         size_t poolIndex = query->getQueryPoolIndex();
-        ASSERT(query->getQueryPool()->valid() && poolIndex < mQueryPoolStats.size() &&
-               mQueryPoolStats[poolIndex].freedCount < mPoolSize);
+        ASSERT(query->getQueryPool()->valid());
 
-        ++mQueryPoolStats[poolIndex].freedCount;
+        onEntryFreed(context, poolIndex);
         query->deinit();
-
-        // Take note of the current serial to avoid reallocating a query in the same pool
-        mQueryPoolStats[poolIndex].serial = context->getRenderer()->getCurrentQueueSerial();
     }
 }
 
 angle::Result DynamicQueryPool::allocateNewPool(Context *context)
 {
-    // Before allocating a new pool, see if a previously allocated pool is completely freed up.
-    // In that case, use that older pool instead.
-    Serial lastCompletedQueueSerial = context->getRenderer()->getLastCompletedQueueSerial();
-    for (size_t i = 0; i < mQueryPools.size(); ++i)
+    if (findFreeEntryPool(context))
     {
-        if (mQueryPoolStats[i].freedCount == mPoolSize &&
-            mQueryPoolStats[i].serial <= lastCompletedQueueSerial)
-        {
-            mCurrentQueryPool = i;
-            mCurrentFreeQuery = 0;
-
-            return angle::Result::Continue();
-        }
+        return angle::Result::Continue();
     }
 
     VkQueryPoolCreateInfo queryPoolInfo = {};
@@ -505,15 +558,7 @@ angle::Result DynamicQueryPool::allocateNewPool(Context *context)
 
     ANGLE_TRY(queryPool.init(context, queryPoolInfo));
 
-    mQueryPools.push_back(std::move(queryPool));
-
-    QueryPoolStats poolStats = {0, Serial()};
-    mQueryPoolStats.push_back(poolStats);
-
-    mCurrentQueryPool = mQueryPools.size() - 1;
-    mCurrentFreeQuery = 0;
-
-    return angle::Result::Continue();
+    return allocateNewEntryPool(context, std::move(queryPool));
 }
 
 // QueryHelper implementation
@@ -545,6 +590,115 @@ void QueryHelper::deinit()
     mQuery            = 0;
 }
 
+// DynamicSemaphorePool implementation
+DynamicSemaphorePool::DynamicSemaphorePool() = default;
+
+DynamicSemaphorePool::~DynamicSemaphorePool() = default;
+
+angle::Result DynamicSemaphorePool::init(Context *context, uint32_t poolSize)
+{
+    ANGLE_TRY(initEntryPool(context, poolSize));
+    ANGLE_TRY(allocateNewPool(context));
+    return angle::Result::Continue();
+}
+
+void DynamicSemaphorePool::destroy(VkDevice device)
+{
+    for (auto &semaphorePool : mPools)
+    {
+        for (Semaphore &semaphore : semaphorePool)
+        {
+            semaphore.destroy(device);
+        }
+    }
+
+    destroyEntryPool();
+}
+
+angle::Result DynamicSemaphorePool::allocateSemaphore(Context *context,
+                                                      SemaphoreHelper *semaphoreOut)
+{
+    ASSERT(!semaphoreOut->getSemaphore());
+
+    if (mCurrentFreeEntry >= mPoolSize)
+    {
+        // No more queries left in this pool, create another one.
+        ANGLE_TRY(allocateNewPool(context));
+    }
+
+    semaphoreOut->init(mCurrentPool, &mPools[mCurrentPool][mCurrentFreeEntry++]);
+
+    return angle::Result::Continue();
+}
+
+void DynamicSemaphorePool::freeSemaphore(Context *context, SemaphoreHelper *semaphore)
+{
+    if (semaphore->getSemaphore())
+    {
+        onEntryFreed(context, semaphore->getSemaphorePoolIndex());
+        semaphore->deinit();
+    }
+}
+
+angle::Result DynamicSemaphorePool::allocateNewPool(Context *context)
+{
+    if (findFreeEntryPool(context))
+    {
+        return angle::Result::Continue();
+    }
+
+    std::vector<Semaphore> newPool(mPoolSize);
+
+    for (Semaphore &semaphore : newPool)
+    {
+        ANGLE_TRY(semaphore.init(context));
+    }
+
+    // This code is safe as long as the growth of the outer vector in vector<vector<T>> is done by
+    // moving the inner vectors, making sure references to the inner vector remain intact.
+    Semaphore *assertMove = mPools.size() > 0 ? mPools[0].data() : nullptr;
+
+    ANGLE_TRY(allocateNewEntryPool(context, std::move(newPool)));
+
+    ASSERT(assertMove == nullptr || assertMove == mPools[0].data());
+
+    return angle::Result::Continue();
+}
+
+// SemaphoreHelper implementation
+SemaphoreHelper::SemaphoreHelper() : mSemaphorePoolIndex(0), mSemaphore(0)
+{
+}
+
+SemaphoreHelper::~SemaphoreHelper()
+{
+}
+
+SemaphoreHelper::SemaphoreHelper(SemaphoreHelper &&other)
+    : mSemaphorePoolIndex(other.mSemaphorePoolIndex), mSemaphore(other.mSemaphore)
+{
+    other.mSemaphore = nullptr;
+}
+
+SemaphoreHelper &SemaphoreHelper::operator=(SemaphoreHelper &&other)
+{
+    std::swap(mSemaphorePoolIndex, other.mSemaphorePoolIndex);
+    std::swap(mSemaphore, other.mSemaphore);
+    return *this;
+}
+
+void SemaphoreHelper::init(const size_t semaphorePoolIndex, const vk::Semaphore *semaphore)
+{
+    mSemaphorePoolIndex = semaphorePoolIndex;
+    mSemaphore          = semaphore;
+}
+
+void SemaphoreHelper::deinit()
+{
+    mSemaphorePoolIndex = 0;
+    mSemaphore          = nullptr;
+}
+
 // LineLoopHelper implementation.
 LineLoopHelper::LineLoopHelper(RendererVk *renderer)
     : mDynamicIndexBuffer(kLineLoopDynamicBufferUsage, kLineLoopDynamicBufferMinSize)

src/libANGLE/renderer/vulkan/vk_helpers.h

@@ -160,6 +160,54 @@ class DynamicDescriptorPool final : angle::NonCopyable
     VkDescriptorPoolSize mPoolSize;
 };
 
+template <typename Pool>
+class DynamicallyGrowingPool : angle::NonCopyable
+{
+  public:
+    DynamicallyGrowingPool();
+    virtual ~DynamicallyGrowingPool();
+
+    bool isValid() { return mPoolSize > 0; }
+
+  protected:
+    angle::Result initEntryPool(Context *context, uint32_t poolSize);
+    void destroyEntryPool();
+
+    // Checks to see if any pool is already free, in which case it sets it as current pool and
+    // returns true.
+    bool findFreeEntryPool(Context *context);
+
+    // Allocates a new entry and initializes it with the given pool.
+    angle::Result allocateNewEntryPool(Context *context, Pool &&pool);
+
+    // Called by the implementation whenever an entry is freed.
+    void onEntryFreed(Context *context, size_t poolIndex);
+
+    // The pool size, to know when a pool is completely freed.
+    uint32_t mPoolSize;
+
+    std::vector<Pool> mPools;
+
+    struct PoolStats
+    {
+        // A count corresponding to each pool indicating how many of its allocated entries
+        // have been freed. Once that value reaches mPoolSize for each pool, that pool is considered
+        // free and reusable.  While keeping a bitset would allow allocation of each index, the
+        // slight runtime overhead of finding free indices is not worth the slight memory overhead
+        // of creating new pools when unnecessary.
+        uint32_t freedCount;
+        // The serial of the renderer is stored on each object free to make sure no
+        // new allocations are made from the pool until it's not in use.
+        Serial serial;
+    };
+    std::vector<PoolStats> mPoolStats;
+
+    // Index into mPools indicating pool we are currently allocating from.
+    size_t mCurrentPool;
+    // Index inside mPools[mCurrentPool] indicating which index can be allocated next.
+    uint32_t mCurrentFreeEntry;
+};
+
 // DynamicQueryPool allocates indices out of QueryPool as needed.  Once a QueryPool is exhausted,
 // another is created.  The query pools live permanently, but are recycled as indices get freed.
 
@@ -168,7 +216,7 @@ constexpr uint32_t kDefaultOcclusionQueryPoolSize = 64;
 
 class QueryHelper;
 
-class DynamicQueryPool final : angle::NonCopyable
+class DynamicQueryPool final : public DynamicallyGrowingPool<QueryPool>
 {
   public:
     DynamicQueryPool();
@@ -177,41 +225,16 @@ class DynamicQueryPool final : angle::NonCopyable
     angle::Result init(Context *context, VkQueryType type, uint32_t poolSize);
     void destroy(VkDevice device);
 
-    bool isValid() { return mPoolSize > 0; }
-
     angle::Result allocateQuery(Context *context, QueryHelper *queryOut);
     void freeQuery(Context *context, QueryHelper *query);
 
-    const QueryPool *getQueryPool(size_t index) const { return &mQueryPools[index]; }
+    const QueryPool *getQueryPool(size_t index) const { return &mPools[index]; }
 
   private:
     angle::Result allocateNewPool(Context *context);
 
     // Information required to create new query pools
-    uint32_t mPoolSize;
     VkQueryType mQueryType;
-
-    // A list of query pools to allocate from
-    std::vector<QueryPool> mQueryPools;
-
-    struct QueryPoolStats
-    {
-        // A count corresponding to each query pool indicating how many of its allocated indices
-        // have been freed. Once that value reaches mPoolSize for each pool, that pool is considered
-        // free and reusable.  While keeping a bitset would allow allocation of each index, the
-        // slight runtime overhead of finding free indices is not worth the slight memory overhead
-        // of creating new pools when unnecessary.
-        uint32_t freedCount;
-        // When the pool is completely emptied, the serial of the renderer is stored to make sure no
-        // new allocations are made from the pool until it's not in use.
-        Serial serial;
-    };
-    std::vector<QueryPoolStats> mQueryPoolStats;
-
-    // Index into mQueryPools indicating query pool we are currently allocating from
-    size_t mCurrentQueryPool;
-    // Bit index inside mQueryPools[mCurrentQueryPool] indicating which index can be allocated next
-    uint32_t mCurrentFreeQuery;
 };
 
 // Queries in vulkan are identified by the query pool and an index for a query within that pool.
@@ -249,6 +272,59 @@ class QueryHelper final : public CommandGraphResource
     uint32_t mQuery;
 };
 
+// DynamicSemaphorePool allocates semaphores as needed.  It uses a std::vector
+// as a pool to allocate many semaphores at once.  The pools live permanently,
+// but are recycled as semaphores get freed.
+
+// These are arbitrary default sizes for semaphore pools.
+constexpr uint32_t kDefaultSemaphorePoolSize = 64;
+
+class SemaphoreHelper;
+
+class DynamicSemaphorePool final : public DynamicallyGrowingPool<std::vector<Semaphore>>
+{
+  public:
+    DynamicSemaphorePool();
+    ~DynamicSemaphorePool();
+
+    angle::Result init(Context *context, uint32_t poolSize);
+    void destroy(VkDevice device);
+
+    bool isValid() { return mPoolSize > 0; }
+
+    // autoFree can be used to allocate a semaphore that's expected to be freed at the end of the
+    // frame.  This renders freeSemaphore unnecessary and saves an eventual search.
+    angle::Result allocateSemaphore(Context *context, SemaphoreHelper *semaphoreOut);
+    void freeSemaphore(Context *context, SemaphoreHelper *semaphore);
+
+  private:
+    angle::Result allocateNewPool(Context *context);
+};
+
+// Semaphores that are allocated from the semaphore pool are encapsulated in a helper object,
+// keeping track of where in the pool they are allocated from.
+class SemaphoreHelper final : angle::NonCopyable
+{
+  public:
+    SemaphoreHelper();
+    ~SemaphoreHelper();
+
+    SemaphoreHelper(SemaphoreHelper &&other);
+    SemaphoreHelper &operator=(SemaphoreHelper &&other);
+
+    void init(const size_t semaphorePoolIndex, const Semaphore *semaphore);
+    void deinit();
+
+    const Semaphore *getSemaphore() const { return mSemaphore; }
+
+    // Used only by DynamicSemaphorePool.
+    size_t getSemaphorePoolIndex() const { return mSemaphorePoolIndex; }
+
+  private:
+    size_t mSemaphorePoolIndex;
+    const Semaphore *mSemaphore;
+};
+
 // This class' responsibility is to create index buffers needed to support line loops in Vulkan.
 // In the setup phase of drawing, the createIndexBuffer method should be called with the
 // current draw call parameters. If an element array buffer is bound for an indexed draw, use