Vulkan: Add command stream diagnostics when graph is disabled.

src/libANGLE/renderer/vulkan/ContextVk.cpp

@@ -41,6 +41,8 @@
 
 #include "libANGLE/trace.h"
 
+#include <iostream>
+
 namespace rx
 {
 
@@ -107,8 +109,8 @@ constexpr size_t kDriverUniformsAllocatorPageSize = 4 * 1024;
 
 constexpr size_t kInFlightCommandsLimit = 100u;
 
-// Initially dumping the command graphs is disabled.
-constexpr bool kEnableCommandGraphDiagnostics = false;
+// Dumping the command stream is disabled by default.
+constexpr bool kEnableCommandStreamDiagnostics = false;
 
 void InitializeSubmitInfo(VkSubmitInfo *submitInfo,
                           const vk::PrimaryCommandBuffer &commandBuffer,
@@ -175,6 +177,29 @@ void ApplySampleCoverage(const gl::State &glState,
     *maskOut &= coverageMask;
 }
 
+char GetLoadOpShorthand(uint32_t loadOp)
+{
+    switch (loadOp)
+    {
+        case VK_ATTACHMENT_LOAD_OP_CLEAR:
+            return 'C';
+        case VK_ATTACHMENT_LOAD_OP_LOAD:
+            return 'L';
+        default:
+            return 'D';
+    }
+}
+
+char GetStoreOpShorthand(uint32_t storeOp)
+{
+    switch (storeOp)
+    {
+        case VK_ATTACHMENT_STORE_OP_STORE:
+            return 'S';
+        default:
+            return 'D';
+    }
+}
 }  // anonymous namespace
 
 ContextVk::DriverUniformsDescriptorSet::DriverUniformsDescriptorSet()
@@ -506,7 +531,7 @@ ContextVk::ContextVk(const gl::State &state, gl::ErrorSet *errorSet, RendererVk
       mEmulateSeamfulCubeMapSampling(false),
       mUseOldRewriteStructSamplers(false),
       mPoolAllocator(kDefaultPoolAllocatorPageSize, 1),
-      mCommandGraph(kEnableCommandGraphDiagnostics, &mPoolAllocator),
+      mCommandGraph(kEnableCommandStreamDiagnostics, &mPoolAllocator),
       mGpuEventsEnabled(false),
       mGpuClockSync{std::numeric_limits<double>::max(), std::numeric_limits<double>::max()},
       mGpuEventTimestampOrigin(0),
@@ -1020,7 +1045,7 @@ angle::Result ContextVk::setupDispatch(const gl::Context *context,
         // |setupDispatch| and |setupDraw| are special in that they flush dirty bits. Therefore they
         // don't use the same APIs to record commands as the functions outside ContextVk.
         // The following ensures prior commands are flushed before we start processing dirty bits.
-        mOutsideRenderPassCommands.flushToPrimary(&mPrimaryCommands);
+        mOutsideRenderPassCommands.flushToPrimary(this, &mPrimaryCommands);
         ANGLE_TRY(endRenderPass());
         *commandBufferOut = &mOutsideRenderPassCommands.getCommandBuffer();
     }
@@ -1423,6 +1448,11 @@ angle::Result ContextVk::submitFrame(const VkSubmitInfo &submitInfo,
             mState.getOverlay()->getRunningGraphWidget(gl::WidgetId::VulkanRenderPassCount);
         renderPassCount->add(mRenderPassCommands.getAndResetCounter());
         renderPassCount->next();
+
+        if (kEnableCommandStreamDiagnostics)
+        {
+            dumpCommandStreamDiagnostics();
+        }
     }
 
     ANGLE_TRY(ensureSubmitFenceInitialized());
@@ -2222,7 +2252,7 @@ angle::Result ContextVk::clearWithRenderPassOp(
     }
     else
     {
-        mOutsideRenderPassCommands.flushToPrimary(&mPrimaryCommands);
+        mOutsideRenderPassCommands.flushToPrimary(this, &mPrimaryCommands);
     }
 
     size_t attachmentIndexVk = 0;
@@ -3699,7 +3729,7 @@ angle::Result ContextVk::flushImpl(const vk::Semaphore *signalSemaphore)
     }
     else
     {
-        mOutsideRenderPassCommands.flushToPrimary(&mPrimaryCommands);
+        mOutsideRenderPassCommands.flushToPrimary(this, &mPrimaryCommands);
         ANGLE_TRY(endRenderPass());
 
         if (mIsAnyHostVisibleBufferWritten)
@@ -4024,7 +4054,7 @@ angle::Result ContextVk::onBufferRead(VkAccessFlags readAccessType, vk::BufferHe
 
     if (!buffer->canAccumulateRead(this, readAccessType))
     {
-        mOutsideRenderPassCommands.flushToPrimary(&mPrimaryCommands);
+        mOutsideRenderPassCommands.flushToPrimary(this, &mPrimaryCommands);
     }
 
     mOutsideRenderPassCommands.bufferRead(&mResourceUseList, readAccessType, buffer);
@@ -4040,7 +4070,7 @@ angle::Result ContextVk::onBufferWrite(VkAccessFlags writeAccessType, vk::Buffer
 
     if (!buffer->canAccumulateWrite(this, writeAccessType))
     {
-        mOutsideRenderPassCommands.flushToPrimary(&mPrimaryCommands);
+        mOutsideRenderPassCommands.flushToPrimary(this, &mPrimaryCommands);
     }
 
     mOutsideRenderPassCommands.bufferWrite(&mResourceUseList, writeAccessType, buffer);
@@ -4132,6 +4162,37 @@ angle::Result ContextVk::syncExternalMemory()
     return angle::Result::Continue;
 }
 
+void ContextVk::addCommandBufferDiagnostics(const std::string &commandBufferDiagnostics)
+{
+    mCommandBufferDiagnostics.push_back(commandBufferDiagnostics);
+}
+
+void ContextVk::dumpCommandStreamDiagnostics()
+{
+    std::ostream &out = std::cout;
+
+    if (mCommandBufferDiagnostics.empty())
+        return;
+
+    out << "digraph {\n"
+        << "  node [shape=plaintext fontname=\"Consolas\"]\n";
+
+    for (size_t index = 0; index < mCommandBufferDiagnostics.size(); ++index)
+    {
+        const std::string &payload = mCommandBufferDiagnostics[index];
+        out << "  cb" << index << " [label =\"" << payload << "\"];\n";
+    }
+
+    for (size_t index = 0; index < mCommandBufferDiagnostics.size() - 1; ++index)
+    {
+        out << "  cb" << index << " -> cb" << index + 1 << "\n";
+    }
+
+    mCommandBufferDiagnostics.clear();
+
+    out << "}\n";
+}
+
 CommandBufferHelper::CommandBufferHelper()
     : mImageBarrierSrcStageMask(0),
       mImageBarrierDstStageMask(0),
@@ -4235,11 +4296,25 @@ OutsideRenderPassCommandBuffer::OutsideRenderPassCommandBuffer() = default;
 
 OutsideRenderPassCommandBuffer::~OutsideRenderPassCommandBuffer() = default;
 
-void OutsideRenderPassCommandBuffer::flushToPrimary(vk::PrimaryCommandBuffer *primary)
+void OutsideRenderPassCommandBuffer::flushToPrimary(ContextVk *contextVk,
+                                                    vk::PrimaryCommandBuffer *primary)
 {
     if (empty())
         return;
 
+    if (kEnableCommandStreamDiagnostics)
+    {
+        std::ostringstream out;
+        if (mGlobalMemoryBarrierSrcAccess != 0 || mGlobalMemoryBarrierDstAccess != 0)
+        {
+            out << "Memory Barrier Src: 0x" << std::hex << mGlobalMemoryBarrierSrcAccess
+                << " &rarr; Dst: 0x" << std::hex << mGlobalMemoryBarrierDstAccess << "\\l";
+        }
+
+        out << mCommandBuffer.dumpCommands("\\l");
+        contextVk->addCommandBufferDiagnostics(out.str());
+    }
+
     executeBarriers(primary);
     mCommandBuffer.executeCommands(primary->getHandle());
 
@@ -4311,6 +4386,11 @@ angle::Result RenderPassCommandBuffer::flushToPrimary(ContextVk *contextVk,
     if (empty())
         return angle::Result::Continue;
 
+    if (kEnableCommandStreamDiagnostics)
+    {
+        addRenderPassCommandDiagnostics(contextVk);
+    }
+
     executeBarriers(primary);
 
     // Pull a RenderPass from the cache.
@@ -4377,6 +4457,58 @@ angle::Result RenderPassCommandBuffer::flushToPrimary(ContextVk *contextVk,
     return angle::Result::Continue;
 }
 
+void RenderPassCommandBuffer::addRenderPassCommandDiagnostics(ContextVk *contextVk)
+{
+    std::ostringstream out;
+    if (mGlobalMemoryBarrierSrcAccess != 0 || mGlobalMemoryBarrierDstAccess != 0)
+    {
+        out << "Memory Barrier Src: 0x" << std::hex << mGlobalMemoryBarrierSrcAccess
+            << " &rarr; Dst: 0x" << std::hex << mGlobalMemoryBarrierDstAccess << "\\l";
+    }
+
+    size_t attachmentCount             = mRenderPassDesc.attachmentCount();
+    size_t depthStencilAttachmentCount = mRenderPassDesc.hasDepthStencilAttachment();
+    size_t colorAttachmentCount        = attachmentCount - depthStencilAttachmentCount;
+
+    std::string loadOps, storeOps;
+
+    if (colorAttachmentCount > 0)
+    {
+        loadOps += " Color: ";
+        storeOps += " Color: ";
+
+        for (size_t i = 0; i < colorAttachmentCount; ++i)
+        {
+            loadOps += GetLoadOpShorthand(mAttachmentOps[i].loadOp);
+            storeOps += GetStoreOpShorthand(mAttachmentOps[i].storeOp);
+        }
+    }
+
+    if (depthStencilAttachmentCount > 0)
+    {
+        ASSERT(depthStencilAttachmentCount == 1);
+
+        loadOps += " Depth/Stencil: ";
+        storeOps += " Depth/Stencil: ";
+        size_t dsIndex = colorAttachmentCount;
+
+        loadOps += GetLoadOpShorthand(mAttachmentOps[dsIndex].loadOp);
+        loadOps += GetLoadOpShorthand(mAttachmentOps[dsIndex].stencilLoadOp);
+
+        storeOps += GetStoreOpShorthand(mAttachmentOps[dsIndex].storeOp);
+        storeOps += GetStoreOpShorthand(mAttachmentOps[dsIndex].stencilStoreOp);
+    }
+
+    if (attachmentCount > 0)
+    {
+        out << "LoadOp:  " << loadOps << "\\l";
+        out << "StoreOp: " << storeOps << "\\l";
+    }
+
+    out << mCommandBuffer.dumpCommands("\\l");
+    contextVk->addCommandBufferDiagnostics(out.str());
+}
+
 void RenderPassCommandBuffer::reset()
 {
     mCommandBuffer.reset();

src/libANGLE/renderer/vulkan/ContextVk.h

@@ -142,7 +142,7 @@ class OutsideRenderPassCommandBuffer final : public CommandBufferHelper
     OutsideRenderPassCommandBuffer();
     ~OutsideRenderPassCommandBuffer();
 
-    void flushToPrimary(vk::PrimaryCommandBuffer *primary);
+    void flushToPrimary(ContextVk *contextVk, vk::PrimaryCommandBuffer *primary);
 
     bool empty() const { return mCommandBuffer.empty(); }
     void reset();
@@ -216,6 +216,8 @@ class RenderPassCommandBuffer final : public CommandBufferHelper
     }
 
   private:
+    void addRenderPassCommandDiagnostics(ContextVk *contextVk);
+
     uint32_t mCounter;
     vk::RenderPassDesc mRenderPassDesc;
     vk::AttachmentOpsArray mAttachmentOps;
@@ -621,7 +623,7 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
 
     angle::Result getPrimaryCommandBuffer(vk::PrimaryCommandBuffer **primaryCommands)
     {
-        mOutsideRenderPassCommands.flushToPrimary(&mPrimaryCommands);
+        mOutsideRenderPassCommands.flushToPrimary(this, &mPrimaryCommands);
         ANGLE_TRY(endRenderPass());
         *primaryCommands = &mPrimaryCommands;
         return angle::Result::Continue;
@@ -632,6 +634,8 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
 
     angle::Result syncExternalMemory();
 
+    void addCommandBufferDiagnostics(const std::string &commandBufferDiagnostics);
+
   private:
     // Dirty bits.
     enum DirtyBitType : size_t
@@ -885,6 +889,7 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
     angle::Result ensureSubmitFenceInitialized();
     angle::Result startPrimaryCommandBuffer();
     bool hasRecordedCommands();
+    void dumpCommandStreamDiagnostics();
 
     std::array<DirtyBitHandler, DIRTY_BIT_MAX> mGraphicsDirtyBitHandlers;
     std::array<DirtyBitHandler, DIRTY_BIT_MAX> mComputeDirtyBitHandlers;
@@ -1034,6 +1039,8 @@ class ContextVk : public ContextImpl, public vk::Context, public vk::RenderPassO
     vk::ResourceUseList mResourceUseList;
 
     egl::ContextPriority mContextPriority;
+
+    std::vector<std::string> mCommandBufferDiagnostics;
 };
 }  // namespace rx
 

src/libANGLE/renderer/vulkan/SecondaryCommandBuffer.cpp

@@ -17,6 +17,103 @@ namespace vk
 {
 namespace priv
 {
+namespace
+{
+const char *GetCommandString(CommandID id)
+{
+    switch (id)
+    {
+        case CommandID::Invalid:
+            return "--Invalid--";
+        case CommandID::BeginQuery:
+            return "BeginQuery";
+        case CommandID::BindComputePipeline:
+            return "BindComputePipeline";
+        case CommandID::BindDescriptorSets:
+            return "BindDescriptorSets";
+        case CommandID::BindGraphicsPipeline:
+            return "BindGraphicsPipeline";
+        case CommandID::BindIndexBuffer:
+            return "BindIndexBuffer";
+        case CommandID::BindVertexBuffers:
+            return "BindVertexBuffers";
+        case CommandID::BindTransformFeedbackBuffers:
+            return "BindTransformFeedbackBuffers";
+        case CommandID::BlitImage:
+            return "BlitImage";
+        case CommandID::BufferBarrier:
+            return "BufferBarrier";
+        case CommandID::ClearAttachments:
+            return "ClearAttachments";
+        case CommandID::ClearColorImage:
+            return "ClearColorImage";
+        case CommandID::ClearDepthStencilImage:
+            return "ClearDepthStencilImage";
+        case CommandID::CopyBuffer:
+            return "CopyBuffer";
+        case CommandID::CopyBufferToImage:
+            return "CopyBufferToImage";
+        case CommandID::CopyImage:
+            return "CopyImage";
+        case CommandID::CopyImageToBuffer:
+            return "CopyImageToBuffer";
+        case CommandID::Dispatch:
+            return "Dispatch";
+        case CommandID::DispatchIndirect:
+            return "DispatchIndirect";
+        case CommandID::Draw:
+            return "Draw";
+        case CommandID::DrawIndexed:
+            return "DrawIndexed";
+        case CommandID::DrawIndexedBaseVertex:
+            return "DrawIndexedBaseVertex";
+        case CommandID::DrawIndexedInstanced:
+            return "DrawIndexedInstanced";
+        case CommandID::DrawIndexedInstancedBaseVertex:
+            return "DrawIndexedInstancedBaseVertex";
+        case CommandID::DrawIndexedInstancedBaseVertexBaseInstance:
+            return "DrawIndexedInstancedBaseVertexBaseInstance";
+        case CommandID::DrawInstanced:
+            return "DrawInstanced";
+        case CommandID::DrawInstancedBaseInstance:
+            return "DrawInstancedBaseInstance";
+        case CommandID::DrawIndexedIndirect:
+            return "DrawIndexedIndirect";
+        case CommandID::DrawIndirect:
+            return "DrawIndirect";
+        case CommandID::EndQuery:
+            return "EndQuery";
+        case CommandID::ExecutionBarrier:
+            return "ExecutionBarrier";
+        case CommandID::FillBuffer:
+            return "FillBuffer";
+        case CommandID::ImageBarrier:
+            return "ImageBarrier";
+        case CommandID::MemoryBarrier:
+            return "MemoryBarrier";
+        case CommandID::PipelineBarrier:
+            return "PipelineBarrier";
+        case CommandID::PushConstants:
+            return "PushConstants";
+        case CommandID::ResetEvent:
+            return "ResetEvent";
+        case CommandID::ResetQueryPool:
+            return "ResetQueryPool";
+        case CommandID::ResolveImage:
+            return "ResolveImage";
+        case CommandID::SetEvent:
+            return "SetEvent";
+        case CommandID::WaitEvents:
+            return "WaitEvents";
+        case CommandID::WriteTimestamp:
+            return "WriteTimestamp";
+        default:
+            // Need this to work around MSVC warning 4715.
+            UNREACHABLE();
+            return "--unreachable--";
+    }
+}
+}  // namespace
 
 ANGLE_INLINE const CommandHeader *NextCommand(const CommandHeader *command)
 {
@@ -419,139 +516,16 @@ void SecondaryCommandBuffer::getMemoryUsageStats(size_t *usedMemoryOut,
 
 std::string SecondaryCommandBuffer::dumpCommands(const char *separator) const
 {
-    std::string result;
+    std::stringstream result;
     for (const CommandHeader *command : mCommands)
     {
         for (const CommandHeader *currentCommand                      = command;
              currentCommand->id != CommandID::Invalid; currentCommand = NextCommand(currentCommand))
         {
-            result += separator;
-            switch (currentCommand->id)
-            {
-                case CommandID::BeginQuery:
-                    result += "BeginQuery";
-                    break;
-                case CommandID::BindComputePipeline:
-                    result += "BindComputePipeline";
-                    break;
-                case CommandID::BindDescriptorSets:
-                    result += "BindDescriptorSets";
-                    break;
-                case CommandID::BindGraphicsPipeline:
-                    result += "BindGraphicsPipeline";
-                    break;
-                case CommandID::BindIndexBuffer:
-                    result += "BindIndexBuffer";
-                    break;
-                case CommandID::BindVertexBuffers:
-                    result += "BindVertexBuffers";
-                    break;
-                case CommandID::BindTransformFeedbackBuffers:
-                    result += "BindTransformFeedbackBuffers";
-                    break;
-                case CommandID::BlitImage:
-                    result += "BlitImage";
-                    break;
-                case CommandID::BufferBarrier:
-                    result += "BufferBarrier";
-                    break;
-                case CommandID::ClearAttachments:
-                    result += "ClearAttachments";
-                    break;
-                case CommandID::ClearColorImage:
-                    result += "ClearColorImage";
-                    break;
-                case CommandID::ClearDepthStencilImage:
-                    result += "ClearDepthStencilImage";
-                    break;
-                case CommandID::CopyBuffer:
-                    result += "CopyBuffer";
-                    break;
-                case CommandID::CopyBufferToImage:
-                    result += "CopyBufferToImage";
-                    break;
-                case CommandID::CopyImage:
-                    result += "CopyImage";
-                    break;
-                case CommandID::CopyImageToBuffer:
-                    result += "CopyImageToBuffer";
-                    break;
-                case CommandID::Dispatch:
-                    result += "Dispatch";
-                    break;
-                case CommandID::DispatchIndirect:
-                    result += "DispatchIndirect";
-                    break;
-                case CommandID::Draw:
-                    result += "Draw";
-                    break;
-                case CommandID::DrawIndexed:
-                    result += "DrawIndexed";
-                    break;
-                case CommandID::DrawIndexedBaseVertex:
-                    result += "DrawIndexedBaseVertex";
-                    break;
-                case CommandID::DrawIndexedInstanced:
-                    result += "DrawIndexedInstanced";
-                    break;
-                case CommandID::DrawIndexedInstancedBaseVertex:
-                    result += "DrawIndexedInstancedBaseVertex";
-                    break;
-                case CommandID::DrawInstanced:
-                    result += "DrawInstanced";
-                    break;
-                case CommandID::DrawIndexedIndirect:
-                    result += "DrawIndexedIndirect";
-                    break;
-                case CommandID::EndQuery:
-                    result += "EndQuery";
-                    break;
-                case CommandID::ExecutionBarrier:
-                    result += "ExecutionBarrier";
-                    break;
-                case CommandID::FillBuffer:
-                    result += "FillBuffer";
-                    break;
-                case CommandID::ImageBarrier:
-                    result += "ImageBarrier";
-                    break;
-                case CommandID::MemoryBarrier:
-                    result += "MemoryBarrier";
-                    break;
-                case CommandID::PipelineBarrier:
-                    result += "PipelineBarrier";
-                    break;
-                case CommandID::PushConstants:
-                    result += "PushConstants";
-                    break;
-                case CommandID::ResetEvent:
-                    result += "ResetEvent";
-                    break;
-                case CommandID::ResetQueryPool:
-                    result += "ResetQueryPool";
-                    break;
-                case CommandID::ResolveImage:
-                    result += "ResolveImage";
-                    break;
-                case CommandID::SetEvent:
-                    result += "SetEvent";
-                    break;
-                case CommandID::WaitEvents:
-                    result += "WaitEvents";
-                    break;
-                case CommandID::WriteTimestamp:
-                    result += "WriteTimestamp";
-                    break;
-                default:
-                {
-                    UNREACHABLE();
-                    result += "--invalid--";
-                    break;
-                }
-            }
+            result << GetCommandString(currentCommand->id) << separator;
         }
     }
-    return result;
+    return result.str();
 }
 
 }  // namespace priv