D3D: Add a wrapper to Image11 that enables ASAN/MSAN to catch the errors on map operations

src/libANGLE/renderer/d3d/d3d11/Image11.cpp

@@ -520,6 +520,7 @@ angle::Result Image11::getStagingTexture(const gl::Context *context,
 void Image11::releaseStagingTexture()
 {
     mStagingTexture.reset();
+    mStagingTextureSubresourceVerifier.reset();
 }
 
 angle::Result Image11::createStagingTexture(const gl::Context *context)
@@ -577,6 +578,7 @@ angle::Result Image11::createStagingTexture(const gl::Context *context)
 
             mStagingTexture.setDebugName("Image11::StagingTexture3D");
             mStagingSubresource = D3D11CalcSubresource(lodOffset, 0, lodOffset + 1);
+            mStagingTextureSubresourceVerifier.setDesc(desc);
         }
         break;
 
@@ -615,6 +617,7 @@ angle::Result Image11::createStagingTexture(const gl::Context *context)
 
             mStagingTexture.setDebugName("Image11::StagingTexture2D");
             mStagingSubresource = D3D11CalcSubresource(lodOffset, 0, lodOffset + 1);
+            mStagingTextureSubresourceVerifier.setDesc(desc);
         }
         break;
 
@@ -642,6 +645,17 @@ angle::Result Image11::map(const gl::Context *context,
     ANGLE_TRY(
         mRenderer->mapResource(context, stagingTexture->get(), subresourceIndex, mapType, 0, map));
 
+    if (!mStagingTextureSubresourceVerifier.wrap(mapType, map))
+    {
+        ID3D11DeviceContext *deviceContext = mRenderer->getDeviceContext();
+        deviceContext->Unmap(mStagingTexture.get(), mStagingSubresource);
+        Context11 *context11 = GetImplAs<Context11>(context);
+        context11->handleError(GL_OUT_OF_MEMORY,
+                               "Failed to allocate staging texture mapping verifier buffer.",
+                               __FILE__, ANGLE_FUNCTION, __LINE__);
+        return angle::Result::Stop;
+    }
+
     mDirty = true;
 
     return angle::Result::Continue;
@@ -651,6 +665,7 @@ void Image11::unmap()
 {
     if (mStagingTexture.valid())
     {
+        mStagingTextureSubresourceVerifier.unwrap();
         ID3D11DeviceContext *deviceContext = mRenderer->getDeviceContext();
         deviceContext->Unmap(mStagingTexture.get(), mStagingSubresource);
     }

src/libANGLE/renderer/d3d/d3d11/Image11.h

@@ -13,6 +13,7 @@
 #include "common/debug.h"
 #include "libANGLE/ImageIndex.h"
 #include "libANGLE/renderer/d3d/ImageD3D.h"
+#include "libANGLE/renderer/d3d/d3d11/MappedSubresourceVerifier11.h"
 #include "libANGLE/renderer/d3d/d3d11/renderer11_utils.h"
 
 namespace gl
@@ -113,6 +114,7 @@ class Image11 : public ImageD3D
     DXGI_FORMAT mDXGIFormat;
     TextureHelper11 mStagingTexture;
     unsigned int mStagingSubresource;
+    MappedSubresourceVerifier11 mStagingTextureSubresourceVerifier;
 
     bool mRecoverFromStorage;
     TextureStorage11 *mAssociatedStorage;

src/libANGLE/renderer/d3d/d3d11/MappedSubresourceVerifier11.cpp

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+// MappedSubresourceVerifier11.cpp: Implements the
+// rx::MappedSubresourceVerifier11 class, a simple wrapper to D3D11 Texture2D
+// mapped memory so that ASAN and MSAN can catch memory errors done with a
+// pointer to the mapped texture memory.
+
+#include "libANGLE/renderer/d3d/d3d11/MappedSubresourceVerifier11.h"
+
+#include "libANGLE/renderer/d3d/d3d11/formatutils11.h"
+
+namespace rx
+{
+
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ANGLE_ENABLE_ASSERTS)
+
+namespace
+{
+
+#    if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER)
+constexpr bool kUseWrap = true;
+#    else
+constexpr bool kUseWrap = false;
+#    endif
+
+size_t getPitchCount(const D3D11_TEXTURE2D_DESC &desc)
+{
+    const d3d11::DXGIFormatSize &dxgiFormatInfo = d3d11::GetDXGIFormatSizeInfo(desc.Format);
+    ASSERT(desc.Height % dxgiFormatInfo.blockHeight == 0);
+    return desc.Height / dxgiFormatInfo.blockHeight;
+}
+
+}  // namespace
+
+MappedSubresourceVerifier11::MappedSubresourceVerifier11() = default;
+
+MappedSubresourceVerifier11::~MappedSubresourceVerifier11()
+{
+    ASSERT(!mOrigData);
+    ASSERT(!mWrapData.size());
+}
+
+void MappedSubresourceVerifier11::setDesc(const D3D11_TEXTURE2D_DESC &desc)
+{
+    ASSERT(desc.CPUAccessFlags & (D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE));
+    ASSERT(desc.Width);
+    ASSERT(desc.Height);
+    ASSERT(!mOrigData);
+    ASSERT(!mWrapData.size());
+    ASSERT(!mPitchType);
+    ASSERT(!mPitchCount);
+    mPitchType  = &D3D11_MAPPED_SUBRESOURCE::RowPitch;
+    mPitchCount = getPitchCount(desc);
+}
+
+void MappedSubresourceVerifier11::setDesc(const D3D11_TEXTURE3D_DESC &desc)
+{
+    ASSERT(desc.CPUAccessFlags & (D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE));
+    ASSERT(desc.Width);
+    ASSERT(desc.Height);
+    ASSERT(desc.Depth);
+    ASSERT(!mOrigData);
+    ASSERT(!mWrapData.size());
+    ASSERT(!mPitchType);
+    ASSERT(!mPitchCount);
+    mPitchType  = &D3D11_MAPPED_SUBRESOURCE::DepthPitch;
+    mPitchCount = desc.Depth;
+}
+
+void MappedSubresourceVerifier11::reset()
+{
+    ASSERT(!mOrigData);
+    ASSERT(!mWrapData.size());
+    mPitchType  = nullptr;
+    mPitchCount = 0;
+}
+
+bool MappedSubresourceVerifier11::wrap(D3D11_MAP mapType, D3D11_MAPPED_SUBRESOURCE *map)
+{
+    ASSERT(map && map->pData);
+    ASSERT(mapType == D3D11_MAP_READ || mapType == D3D11_MAP_WRITE ||
+           mapType == D3D11_MAP_READ_WRITE);
+    ASSERT(mPitchCount);
+
+    if (kUseWrap)
+    {
+        if (!mWrapData.resize(mPitchCount * map->*mPitchType))
+            return false;
+    }
+
+    mOrigData = reinterpret_cast<uint8_t *>(map->pData);
+
+    if (kUseWrap)
+    {
+        std::copy(mOrigData, mOrigData + mWrapData.size(), mWrapData.data());
+        map->pData = mWrapData.data();
+    }
+    return true;
+}
+
+void MappedSubresourceVerifier11::unwrap()
+{
+    ASSERT(mPitchCount);
+    ASSERT(mOrigData);
+    if (kUseWrap)
+    {
+        std::copy(mWrapData.data(), mWrapData.data() + mWrapData.size(), mOrigData);
+        mWrapData = angle::MemoryBuffer();
+    }
+    mOrigData = nullptr;
+}
+
+#endif
+
+}  // namespace rx

src/libANGLE/renderer/d3d/d3d11/MappedSubresourceVerifier11.h

+//
+// Copyright 2019 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+// MappedSubresourceVerifier11.h: Defines the rx::MappedSubresourceVerifier11
+// class, a simple wrapper to D3D11 Texture2D mapped memory so that ASAN
+// MSAN can catch memory errors done with a pointer to the mapped texture
+// memory.
+
+#ifndef LIBANGLE_RENDERER_D3D_D3D11_MAPPED_SUBRESOURCE_VERIFIER11_H_
+#define LIBANGLE_RENDERER_D3D_D3D11_MAPPED_SUBRESOURCE_VERIFIER11_H_
+
+#include "common/MemoryBuffer.h"
+#include "common/angleutils.h"
+
+namespace rx
+{
+
+class MappedSubresourceVerifier11 final : angle::NonCopyable
+{
+  public:
+    MappedSubresourceVerifier11();
+    ~MappedSubresourceVerifier11();
+
+    void setDesc(const D3D11_TEXTURE2D_DESC &desc);
+    void setDesc(const D3D11_TEXTURE3D_DESC &desc);
+    void reset();
+
+    bool wrap(D3D11_MAP mapType, D3D11_MAPPED_SUBRESOURCE *map);
+    void unwrap();
+
+  private:
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ANGLE_ENABLE_ASSERTS)
+    UINT D3D11_MAPPED_SUBRESOURCE::*mPitchType = nullptr;
+    size_t mPitchCount                         = 0;
+    angle::MemoryBuffer mWrapData;
+    uint8_t *mOrigData = nullptr;
+#endif
+};
+
+#if !(defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || defined(ANGLE_ENABLE_ASSERTS))
+
+inline MappedSubresourceVerifier11::MappedSubresourceVerifier11()  = default;
+inline MappedSubresourceVerifier11::~MappedSubresourceVerifier11() = default;
+
+inline void MappedSubresourceVerifier11::setDesc(const D3D11_TEXTURE2D_DESC &desc) {}
+inline void MappedSubresourceVerifier11::setDesc(const D3D11_TEXTURE3D_DESC &desc) {}
+inline void MappedSubresourceVerifier11::reset() {}
+
+inline bool MappedSubresourceVerifier11::wrap(D3D11_MAP mapType, D3D11_MAPPED_SUBRESOURCE *map)
+{
+    return true;
+}
+
+inline void MappedSubresourceVerifier11::unwrap() {}
+
+#endif
+
+}  // namespace rx
+
+#endif  // LIBANGLE_RENDERER_D3D_D3D11_MAPPED_SUBRESOURCE_VERIFIER11_H_

src/libGLESv2.gni

@@ -492,6 +492,8 @@ libangle_d3d11_sources = [
   "src/libANGLE/renderer/d3d/d3d11/InputLayoutCache.cpp",
   "src/libANGLE/renderer/d3d/d3d11/InputLayoutCache.h",
   "src/libANGLE/renderer/d3d/d3d11/NativeWindow11.h",
+  "src/libANGLE/renderer/d3d/d3d11/MappedSubresourceVerifier11.cpp",
+  "src/libANGLE/renderer/d3d/d3d11/MappedSubresourceVerifier11.h",
   "src/libANGLE/renderer/d3d/d3d11/PixelTransfer11.cpp",
   "src/libANGLE/renderer/d3d/d3d11/PixelTransfer11.h",
   "src/libANGLE/renderer/d3d/d3d11/Program11.cpp",