Skip to content

A
angle
Commit f19a4a20 by Jamie Madill Committed by Commit Bot
Options

Vulkan: Move CommandBuffer management to RendererVk.

This consolidates all relevant logic in a single place. We no longer need to interact with ContextVk in the worker thread. This switches the fixed pointer array size to a dynamically sized vector. Some of the EGL and ANGLE tests would use a large number of Contexts and we were consistently running out of available command buffers which would cause a deadlock situation. We can trust other parts of the code to throttle the application if it starts to get too far ahead of the device and dispense with the hard coded limit in the command buffer allocator itself. The resulting code is also quite a bit simpler and doesn't need a condition variable. Also fixes missing initialization in SecondaryCommandBuffer. Bug: b/172704839 Change-Id: Icc3a3daf5d6b272db556c0e4c93fb793583966a5 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2525143 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarTim Van Patten <timvp@google.com> Reviewed-by: 's avatarCourtney Goeltzenleuchter <courtneygo@google.com>
parent ce7bdd0b
Showing with 70 additions and 96 deletions
src/libANGLE/renderer/vulkan/CommandProcessor.cpp
......@@ -62,7 +62,6 @@ bool CommandsHaveValidOrdering(const std::vector<vk::CommandBatch> &commands)
void CommandProcessorTask::initTask()
{
mTask = CustomTask::Invalid;
mContextVk = nullptr;
mRenderPass = nullptr;
mCommandBuffer = nullptr;
mSemaphore = nullptr;
......@@ -77,12 +76,10 @@ void CommandProcessorTask::initTask()
}
// CommandProcessorTask implementation
void CommandProcessorTask::initProcessCommands(ContextVk *contextVk,
CommandBufferHelper *commandBuffer,
void CommandProcessorTask::initProcessCommands(CommandBufferHelper *commandBuffer,
const RenderPass *renderPass)
{
mTask = CustomTask::ProcessCommands;
mContextVk = contextVk;
mCommandBuffer = commandBuffer;
mRenderPass = renderPass;
}
......@@ -202,7 +199,6 @@ CommandProcessorTask &CommandProcessorTask::operator=(CommandProcessorTask &&rhs
return *this;
}
mContextVk = rhs.mContextVk;
mRenderPass = rhs.mRenderPass;
mCommandBuffer = rhs.mCommandBuffer;
std::swap(mTask, rhs.mTask);
......@@ -467,7 +463,7 @@ angle::Result CommandProcessor::processTask(CommandProcessorTask *task)
ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(this, task->getCommandBuffer()));
}
ASSERT(task->getCommandBuffer()->empty());
task->getCommandBuffer()->releaseToContextQueue(task->getContextVk());
mRenderer->recycleCommandBufferHelper(task->getCommandBuffer());
break;
}
case CustomTask::CheckCompletedCommands:
......
src/libANGLE/renderer/vulkan/CommandProcessor.h
......@@ -63,9 +63,7 @@ class CommandProcessorTask
void initTask(CustomTask command) { mTask = command; }
void initProcessCommands(ContextVk *contextVk,
CommandBufferHelper *commandBuffer,
const RenderPass *renderPass);
void initProcessCommands(CommandBufferHelper *commandBuffer, const RenderPass *renderPass);
void initPresent(egl::ContextPriority priority, VkPresentInfoKHR &presentInfo);
......@@ -106,7 +104,6 @@ class CommandProcessorTask
const VkPresentInfoKHR &getPresentInfo() const { return mPresentInfo; }
const RenderPass *getRenderPass() const { return mRenderPass; }
CommandBufferHelper *getCommandBuffer() const { return mCommandBuffer; }
ContextVk *getContextVk() const { return mContextVk; }
private:
void copyPresentInfo(const VkPresentInfoKHR &other);
......@@ -114,7 +111,6 @@ class CommandProcessorTask
CustomTask mTask;
// ProcessCommands
ContextVk *mContextVk;
const RenderPass *mRenderPass;
CommandBufferHelper *mCommandBuffer;
......
src/libANGLE/renderer/vulkan/ContextVk.cpp
......@@ -505,6 +505,12 @@ void ContextVk::onDestroy(const gl::Context *context)
queryPool.destroy(device);
}
// Recycle current commands buffers.
mRenderer->recycleCommandBufferHelper(mOutsideRenderPassCommands);
mRenderer->recycleCommandBufferHelper(mRenderPassCommands);
mOutsideRenderPassCommands = nullptr;
mRenderPassCommands = nullptr;
ASSERT(mCurrentGarbage.empty());
mRenderer->releaseSharedResources(&mResourceUseList);
......@@ -612,17 +618,9 @@ angle::Result ContextVk::initialize()
mUseOldRewriteStructSamplers = shouldUseOldRewriteStructSamplers();
// Prepare command buffer queue by:
// 1. Initializing each command buffer (as non-renderpass initially)
// 2. Put a pointer to each command buffer into queue
for (vk::CommandBufferHelper &commandBuffer : mCommandBuffers)
{
commandBuffer.initialize(false);
recycleCommandBuffer(&commandBuffer);
}
// Now assign initial command buffers from queue
getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
getNextAvailableCommandBuffer(&mRenderPassCommands, true);
// Assign initial command buffers from queue
mOutsideRenderPassCommands = mRenderer->getCommandBufferHelper(false);
mRenderPassCommands = mRenderer->getCommandBufferHelper(true);
if (mGpuEventsEnabled)
{
......@@ -4548,12 +4546,6 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
ANGLE_TRY(mRenderer->flushRenderPassCommands(this, *renderPass, &mRenderPassCommands));
// TODO(jmadill): Manage in RendererVk. b/172678125
if (getFeatures().commandProcessor.enabled)
{
getNextAvailableCommandBuffer(&mRenderPassCommands, true);
}
if (mGpuEventsEnabled)
{
EventName eventName = GetTraceEventName("RP", mPerfCounters.renderPasses);
......@@ -4571,31 +4563,6 @@ angle::Result ContextVk::flushCommandsAndEndRenderPass()
return angle::Result::Continue;
}
void ContextVk::getNextAvailableCommandBuffer(vk::CommandBufferHelper **commandBuffer,
bool hasRenderPass)
{
ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::getNextAvailableCommandBuffer");
std::unique_lock<std::mutex> lock(mCommandBufferQueueMutex);
// Only wake if notified and command queue is not empty
mAvailableCommandBufferCondition.wait(lock,
[this] { return !mAvailableCommandBuffers.empty(); });
*commandBuffer = mAvailableCommandBuffers.front();
ASSERT((*commandBuffer)->empty());
mAvailableCommandBuffers.pop();
lock.unlock();
(*commandBuffer)->setHasRenderPass(hasRenderPass);
(*commandBuffer)->markOpen();
}
void ContextVk::recycleCommandBuffer(vk::CommandBufferHelper *commandBuffer)
{
ANGLE_TRACE_EVENT0("gpu.angle", "ContextVk::recycleCommandBuffer");
std::lock_guard<std::mutex> queueLock(mCommandBufferQueueMutex);
ASSERT(commandBuffer->empty());
mAvailableCommandBuffers.push(commandBuffer);
mAvailableCommandBufferCondition.notify_one();
}
angle::Result ContextVk::syncExternalMemory()
{
VkMemoryBarrier memoryBarrier = {};
......@@ -4687,12 +4654,6 @@ angle::Result ContextVk::flushOutsideRenderPassCommands()
ANGLE_TRY(mRenderer->flushOutsideRPCommands(this, &mOutsideRenderPassCommands));
// TODO(jmadill): Manage in RendererVk. b/172678125
if (getFeatures().commandProcessor.enabled)
{
getNextAvailableCommandBuffer(&mOutsideRenderPassCommands, false);
}
mPerfCounters.flushedOutsideRenderPassCommandBuffers++;
return angle::Result::Continue;
}
......
src/libANGLE/renderer/vulkan/ContextVk.h
......@@ -592,9 +592,6 @@ class ContextVk : public ContextImpl, public vk::Context
void updateOverlayOnPresent();
void addOverlayUsedBuffersCount(vk::CommandBufferHelper *commandBuffer);
// When worker thread completes, it releases command buffers back to context queue
void recycleCommandBuffer(vk::CommandBufferHelper *commandBuffer);
// DescriptorSet writes
VkDescriptorBufferInfo *allocDescriptorBufferInfos(size_t count);
VkDescriptorImageInfo *allocDescriptorImageInfos(size_t count);
......@@ -904,9 +901,6 @@ class ContextVk : public ContextImpl, public vk::Context
void initIndexTypeMap();
// Pull an available CBH ptr from the CBH queue and set to specified hasRenderPass state
void getNextAvailableCommandBuffer(vk::CommandBufferHelper **commandBuffer, bool hasRenderPass);
angle::Result endRenderPassIfImageUsed(const vk::ImageHelper &image);
angle::Result endRenderPassIfTransformFeedbackBuffer(const vk::BufferHelper *buffer);
......@@ -1031,17 +1025,6 @@ class ContextVk : public ContextImpl, public vk::Context
RenderPassCache mRenderPassCache;
// We have a queue of CommandBufferHelpers (CBHs) that is drawn from for the two active command
// buffers in the main thread. The two active command buffers are the inside and outside
// RenderPass command buffers.
constexpr static size_t kNumCommandBuffers = 50;
std::array<vk::CommandBufferHelper, kNumCommandBuffers> mCommandBuffers;
// Lock access to the command buffer queue
std::mutex mCommandBufferQueueMutex;
std::queue<vk::CommandBufferHelper *> mAvailableCommandBuffers;
std::condition_variable mAvailableCommandBufferCondition;
vk::CommandBufferHelper *mOutsideRenderPassCommands;
vk::CommandBufferHelper *mRenderPassCommands;
......
src/libANGLE/renderer/vulkan/RendererVk.cpp
......@@ -548,6 +548,12 @@ void RendererVk::onDestroy(vk::Context *context)
mSamplerCache.destroy(this);
mYuvConversionCache.destroy(this);
for (vk::CommandBufferHelper *commandBufferHelper : mCommandBufferHelperFreeList)
{
SafeDelete(commandBufferHelper);
}
mCommandBufferHelperFreeList.clear();
mAllocator.destroy();
if (mGlslangInitialized)
......@@ -2666,7 +2672,7 @@ angle::Result RendererVk::checkCompletedCommands(vk::Context *context)
return angle::Result::Continue;
}
angle::Result RendererVk::flushRenderPassCommands(ContextVk *contextVk,
angle::Result RendererVk::flushRenderPassCommands(vk::Context *context,
const vk::RenderPass &renderPass,
vk::CommandBufferHelper **renderPassCommands)
{
......@@ -2675,20 +2681,20 @@ angle::Result RendererVk::flushRenderPassCommands(ContextVk *contextVk,
{
(*renderPassCommands)->markClosed();
vk::CommandProcessorTask flushToPrimary;
flushToPrimary.initProcessCommands(contextVk, *renderPassCommands, &renderPass);
commandProcessorSyncErrorsAndQueueCommand(contextVk, &flushToPrimary);
flushToPrimary.initProcessCommands(*renderPassCommands, &renderPass);
commandProcessorSyncErrorsAndQueueCommand(context, &flushToPrimary);
*renderPassCommands = getCommandBufferHelper(true);
}
else
{
std::lock_guard<std::mutex> lock(mCommandQueueMutex);
ANGLE_TRY(
mCommandQueue.flushRenderPassCommands(contextVk, renderPass, *renderPassCommands));
ANGLE_TRY(mCommandQueue.flushRenderPassCommands(context, renderPass, *renderPassCommands));
}
return angle::Result::Continue;
}
angle::Result RendererVk::flushOutsideRPCommands(ContextVk *contextVk,
angle::Result RendererVk::flushOutsideRPCommands(vk::Context *context,
vk::CommandBufferHelper **outsideRPCommands)
{
ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::flushOutsideRPCommands");
......@@ -2696,16 +2702,46 @@ angle::Result RendererVk::flushOutsideRPCommands(ContextVk *contextVk,
{
(*outsideRPCommands)->markClosed();
vk::CommandProcessorTask flushToPrimary;
flushToPrimary.initProcessCommands(contextVk, *outsideRPCommands, nullptr);
commandProcessorSyncErrorsAndQueueCommand(contextVk, &flushToPrimary);
flushToPrimary.initProcessCommands(*outsideRPCommands, nullptr);
commandProcessorSyncErrorsAndQueueCommand(context, &flushToPrimary);
*outsideRPCommands = getCommandBufferHelper(false);
}
else
{
std::lock_guard<std::mutex> lock(mCommandQueueMutex);
ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(contextVk, *outsideRPCommands));
ANGLE_TRY(mCommandQueue.flushOutsideRPCommands(context, *outsideRPCommands));
}
return angle::Result::Continue;
}
vk::CommandBufferHelper *RendererVk::getCommandBufferHelper(bool hasRenderPass)
{
ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::getCommandBufferHelper");
std::unique_lock<std::mutex> lock(mCommandBufferHelperFreeListMutex);
if (mCommandBufferHelperFreeList.empty())
{
vk::CommandBufferHelper *commandBuffer = new vk::CommandBufferHelper();
commandBuffer->initialize(hasRenderPass);
return commandBuffer;
}
else
{
vk::CommandBufferHelper *commandBuffer = mCommandBufferHelperFreeList.back();
mCommandBufferHelperFreeList.pop_back();
commandBuffer->setHasRenderPass(hasRenderPass);
return commandBuffer;
}
}
void RendererVk::recycleCommandBufferHelper(vk::CommandBufferHelper *commandBuffer)
{
ANGLE_TRACE_EVENT0("gpu.angle", "RendererVk::recycleCommandBufferHelper");
std::lock_guard<std::mutex> lock(mCommandBufferHelperFreeListMutex);
ASSERT(commandBuffer->empty());
commandBuffer->markOpen();
mCommandBufferHelperFreeList.push_back(commandBuffer);
}
} // namespace rx
src/libANGLE/renderer/vulkan/RendererVk.h
......@@ -329,13 +329,15 @@ class RendererVk : angle::NonCopyable
angle::Result finish(vk::Context *context);
angle::Result checkCompletedCommands(vk::Context *context);
// TODO(jmadill): Use vk::Context instead of ContextVk. b/172704839
angle::Result flushRenderPassCommands(ContextVk *contextVk,
angle::Result flushRenderPassCommands(vk::Context *context,
const vk::RenderPass &renderPass,
vk::CommandBufferHelper **renderPassCommands);
angle::Result flushOutsideRPCommands(ContextVk *contextVk,
angle::Result flushOutsideRPCommands(vk::Context *context,
vk::CommandBufferHelper **outsideRPCommands);
vk::CommandBufferHelper *getCommandBufferHelper(bool hasRenderPass);
void recycleCommandBufferHelper(vk::CommandBufferHelper *commandBuffer);
private:
angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
void ensureCapsInitialized() const;
......@@ -452,6 +454,10 @@ class RendererVk : angle::NonCopyable
std::mutex mCommandQueueMutex;
vk::CommandQueue mCommandQueue;
// Command buffer pool management.
std::mutex mCommandBufferHelperFreeListMutex;
std::vector<vk::CommandBufferHelper *> mCommandBufferHelperFreeList;
// Command Processor Thread
vk::CommandProcessor mCommandProcessor;
std::thread mCommandProcessorThread;
......
src/libANGLE/renderer/vulkan/SecondaryCommandBuffer.h
......@@ -811,7 +811,7 @@ class SecondaryCommandBuffer final : angle::NonCopyable
return writePointer + sizeInBytes;
}
// flag to indicate that commandBuffer is open for new commands
// Flag to indicate that commandBuffer is open for new commands. Initially open.
bool mIsOpen;
std::vector<CommandHeader *> mCommands;
......@@ -827,8 +827,9 @@ class SecondaryCommandBuffer final : angle::NonCopyable
};
ANGLE_INLINE SecondaryCommandBuffer::SecondaryCommandBuffer()
: mAllocator(nullptr), mCurrentWritePointer(nullptr), mCurrentBytesRemaining(0)
: mIsOpen(true), mAllocator(nullptr), mCurrentWritePointer(nullptr), mCurrentBytesRemaining(0)
{}
ANGLE_INLINE SecondaryCommandBuffer::~SecondaryCommandBuffer() {}
// begin and insert DebugUtilsLabelEXT funcs share this same function body
......
src/libANGLE/renderer/vulkan/vk_helpers.cpp
......@@ -1305,6 +1305,7 @@ angle::Result CommandBufferHelper::flushToPrimary(const angle::FeaturesVk &featu
{
mCommandBuffer.executeCommands(primary->getHandle());
}
// Restart the command buffer.
reset();
......@@ -1442,11 +1443,6 @@ void CommandBufferHelper::reset()
ASSERT(mRenderPassUsedImages.empty());
}
void CommandBufferHelper::releaseToContextQueue(ContextVk *contextVk)
{
contextVk->recycleCommandBuffer(this);
}
void CommandBufferHelper::resumeTransformFeedback()
{
ASSERT(mIsRenderPassCommandBuffer);
......
src/libANGLE/renderer/vulkan/vk_helpers.h
......@@ -974,7 +974,6 @@ class CommandBufferHelper : angle::NonCopyable
#endif
void reset();
void releaseToContextQueue(ContextVk *contextVk);
// Returns true if we have no work to execute. For renderpass command buffer, even if the
// underlying command buffer is empty, we may still have a renderpass with an empty command
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment