Vulkan: pipeline cache not populated as blob cache is not set

AUTHORS

@@ -31,6 +31,8 @@ LG Electronics, Inc.
 IBM Inc.
 AdaptVis GmbH
 Samsung Electronics, Inc.
+Arm Ltd.
+
 
 Jacek Caban
 Mark Callow

CONTRIBUTORS

@@ -152,3 +152,7 @@ Samsung Electronics, Inc.
  Brandon Schade
  Minkyu Jeong
  Mohan Maiya
+
+Arm Ltd.
+ Fei Yang
+

src/libANGLE/Display.cpp

@@ -1110,10 +1110,10 @@ void Display::notifyDeviceLost()
     mDeviceLost = true;
 }
 
-void Display::setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get)
+egl::Error Display::setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get)
 {
     mBlobCache.setBlobCacheFuncs(set, get);
-    mImplementation->setBlobCacheFuncs(set, get);
+    return mImplementation->setBlobCacheFuncs(set, get);
 }
 
 Error Display::waitClient(const gl::Context *context)

src/libANGLE/Display.h

@@ -155,7 +155,7 @@ class Display final : public LabeledObject, angle::NonCopyable
     bool testDeviceLost();
     void notifyDeviceLost();
 
-    void setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get);
+    Error setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get);
     bool areBlobCacheFuncsSet() const { return mBlobCache.areBlobCacheFuncsSet(); }
     BlobCache &getBlobCache() { return mBlobCache; }
 

src/libANGLE/renderer/DisplayImpl.cpp

@@ -63,4 +63,9 @@ const egl::Caps &DisplayImpl::getCaps() const
     return mCaps;
 }
 
+egl::Error DisplayImpl::setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get)
+{
+    return egl::NoError();
+}
+
 }  // namespace rx

src/libANGLE/renderer/DisplayImpl.h

@@ -88,7 +88,7 @@ class DisplayImpl : public EGLImplFactory
     virtual gl::Version getMaxSupportedESVersion() const                     = 0;
     const egl::Caps &getCaps() const;
 
-    virtual void setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get) {}
+    virtual egl::Error setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get);
 
     const egl::DisplayExtensions &getExtensions() const;
 

src/libANGLE/renderer/gl/egl/DisplayEGL.cpp

@@ -154,12 +154,14 @@ void DisplayEGL::generateCaps(egl::Caps *outCaps) const
     outCaps->textureNPOT = true;  // Since we request GLES >= 2
 }
 
-void DisplayEGL::setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get)
+egl::Error DisplayEGL::setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get)
 {
     if (mEGL->hasExtension("EGL_ANDROID_blob_cache"))
     {
         mEGL->setBlobCacheFuncsANDROID(set, get);
     }
+
+    return egl::NoError();
 }
 
 }  // namespace rx

src/libANGLE/renderer/gl/egl/DisplayEGL.h

@@ -33,7 +33,7 @@ class DisplayEGL : public DisplayGL
 
     std::string getVendorString() const override;
 
-    void setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get) override;
+    egl::Error setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get) override;
 
     virtual void destroyNativeContext(EGLContext context) = 0;
 

src/libANGLE/renderer/vulkan/DisplayVk.cpp

@@ -81,6 +81,11 @@ DeviceImpl *DisplayVk::createDevice()
     return nullptr;
 }
 
+egl::Error DisplayVk::setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get)
+{
+    return angle::ToEGL(mRenderer->onSetBlobCacheFuncs(this), this, EGL_BAD_PARAMETER);
+}
+
 egl::Error DisplayVk::waitClient(const gl::Context *context)
 {
     ANGLE_TRACE_EVENT0("gpu.angle", "DisplayVk::waitClient");

src/libANGLE/renderer/vulkan/DisplayVk.h

@@ -38,6 +38,8 @@ class DisplayVk : public DisplayImpl, public vk::Context
 
     DeviceImpl *createDevice() override;
 
+    egl::Error setBlobCacheFuncs(EGLSetBlobFuncANDROID set, EGLGetBlobFuncANDROID get) override;
+
     egl::Error waitClient(const gl::Context *context) override;
     egl::Error waitNative(const gl::Context *context, EGLint engine) override;
 

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -50,8 +50,8 @@ namespace
 // We currently only allocate 2 uniform buffer per descriptor set, one for the fragment shader and
 // one for the vertex shader.
 constexpr size_t kUniformBufferDescriptorsPerDescriptorSet = 2;
-// Update the pipeline cache every this many swaps (if 60fps, this means every 10 minutes)
-constexpr uint32_t kPipelineCacheVkUpdatePeriod = 10 * 60 * 60;
+// Update the pipeline cache every this many swaps.
+constexpr uint32_t kPipelineCacheVkUpdatePeriod = 60;
 // Wait a maximum of 10s.  If that times out, we declare it a failure.
 constexpr uint64_t kMaxFenceWaitTimeNs = 10'000'000'000llu;
 // Per the Vulkan specification, as long as Vulkan 1.1+ is returned by vkEnumerateInstanceVersion,
@@ -498,7 +498,8 @@ RendererVk::RendererVk()
       mMaxVertexAttribDivisor(1),
       mDevice(VK_NULL_HANDLE),
       mDeviceLost(false),
-      mPipelineCacheVkUpdateTimeout(kPipelineCacheVkUpdatePeriod)
+      mPipelineCacheVkUpdateTimeout(kPipelineCacheVkUpdatePeriod),
+      mPipelineCacheDirty(false)
 {
     VkFormatProperties invalid = {0, 0, kInvalidFormatFeatureFlags};
     mFormatProperties.fill(invalid);
@@ -1015,7 +1016,7 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF
     vkGetDeviceQueue(mDevice, mCurrentQueueFamilyIndex, 0, &mQueue);
 
     // Initialize the vulkan pipeline cache.
-    ANGLE_TRY(initPipelineCache(displayVk));
+    ANGLE_TRY(initPipelineCache(displayVk, &mPipelineCache));
 
     return angle::Result::Continue;
 }
@@ -1269,7 +1270,7 @@ void RendererVk::initPipelineCacheVkKey()
                                hashString.length(), mPipelineCacheVkBlobKey.data());
 }
 
-angle::Result RendererVk::initPipelineCache(DisplayVk *display)
+angle::Result RendererVk::initPipelineCache(DisplayVk *display, vk::PipelineCache *pipelineCache)
 {
     initPipelineCacheVkKey();
 
@@ -1284,7 +1285,21 @@ angle::Result RendererVk::initPipelineCache(DisplayVk *display)
     pipelineCacheCreateInfo.initialDataSize = success ? initialData.size() : 0;
     pipelineCacheCreateInfo.pInitialData    = success ? initialData.data() : nullptr;
 
-    ANGLE_VK_TRY(display, mPipelineCache.init(mDevice, pipelineCacheCreateInfo));
+    ANGLE_VK_TRY(display, pipelineCache->init(mDevice, pipelineCacheCreateInfo));
+    return angle::Result::Continue;
+}
+
+angle::Result RendererVk::onSetBlobCacheFuncs(DisplayVk *display)
+{
+    // We should now recreate the pipeline cache with the blob cache pipeline data.
+    vk::PipelineCache pipelineCache;
+    ANGLE_TRY(initPipelineCache(display, &pipelineCache));
+
+    // Merge the newly created pipeline cache into the existing one.
+    ANGLE_VK_TRY(display,
+                 mPipelineCache.merge(mDevice, mPipelineCache.getHandle(), 1, pipelineCache.ptr()));
+
+    pipelineCache.destroy(mDevice);
     return angle::Result::Continue;
 }
 
@@ -1354,6 +1369,11 @@ angle::Result RendererVk::syncPipelineCacheVk(DisplayVk *displayVk)
     {
         return angle::Result::Continue;
     }
+    if (!mPipelineCacheDirty)
+    {
+        mPipelineCacheVkUpdateTimeout = kPipelineCacheVkUpdatePeriod;
+        return angle::Result::Continue;
+    }
 
     mPipelineCacheVkUpdateTimeout = kPipelineCacheVkUpdatePeriod;
 
@@ -1387,6 +1407,7 @@ angle::Result RendererVk::syncPipelineCacheVk(DisplayVk *displayVk)
     }
 
     displayVk->getBlobCache()->putApplication(mPipelineCacheVkBlobKey, *pipelineCacheData);
+    mPipelineCacheDirty = false;
 
     return angle::Result::Continue;
 }

src/libANGLE/renderer/vulkan/RendererVk.h

@@ -151,13 +151,16 @@ class RendererVk : angle::NonCopyable
     static constexpr size_t kMaxExtensionNames = 200;
     using ExtensionNameList = angle::FixedVector<const char *, kMaxExtensionNames>;
 
+    angle::Result onSetBlobCacheFuncs(DisplayVk *display);
+    void onNewGraphicsPipeline() { mPipelineCacheDirty = true; }
+
   private:
     angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex);
     void ensureCapsInitialized() const;
 
     void initFeatures(const ExtensionNameList &extensions);
     void initPipelineCacheVkKey();
-    angle::Result initPipelineCache(DisplayVk *display);
+    angle::Result initPipelineCache(DisplayVk *display, vk::PipelineCache *pipelineCache);
 
     template <VkFormatFeatureFlags VkFormatProperties::*features>
     VkFormatFeatureFlags getFormatFeatureBits(VkFormat format,
@@ -211,6 +214,7 @@ class RendererVk : angle::NonCopyable
     vk::PipelineCache mPipelineCache;
     egl::BlobCache::Key mPipelineCacheVkBlobKey;
     uint32_t mPipelineCacheVkUpdateTimeout;
+    bool mPipelineCacheDirty;
 
     // A cache of VkFormatProperties as queried from the device over time.
     std::array<VkFormatProperties, vk::kNumVkFormats> mFormatProperties;

src/libANGLE/renderer/vulkan/vk_cache_utils.cpp

@@ -1645,6 +1645,7 @@ angle::Result GraphicsPipelineCache::insertPipeline(
     // This "if" is left here for the benefit of VulkanPipelineCachePerfTest.
     if (context != nullptr)
     {
+        context->getRenderer()->onNewGraphicsPipeline();
         ANGLE_TRY(desc.initializePipeline(context, pipelineCacheVk, compatibleRenderPass,
                                           pipelineLayout, activeAttribLocationsMask, vertexModule,
                                           fragmentModule, &newPipeline));

src/libANGLE/renderer/vulkan/vk_wrapper.h

@@ -478,6 +478,10 @@ class PipelineCache final : public WrappedObject<PipelineCache, VkPipelineCache>
 
     VkResult init(VkDevice device, const VkPipelineCacheCreateInfo &createInfo);
     VkResult getCacheData(VkDevice device, size_t *cacheSize, void *cacheData);
+    VkResult merge(VkDevice device,
+                   VkPipelineCache dstCache,
+                   uint32_t srcCacheCount,
+                   const VkPipelineCache *srcCaches);
 };
 
 class DescriptorSetLayout final : public WrappedObject<DescriptorSetLayout, VkDescriptorSetLayout>
@@ -1237,6 +1241,15 @@ ANGLE_INLINE VkResult PipelineCache::init(VkDevice device,
     return vkCreatePipelineCache(device, &createInfo, nullptr, &mHandle);
 }
 
+ANGLE_INLINE VkResult PipelineCache::merge(VkDevice device,
+                                           VkPipelineCache dstCache,
+                                           uint32_t srcCacheCount,
+                                           const VkPipelineCache *srcCaches)
+{
+    ASSERT(valid());
+    return vkMergePipelineCaches(device, dstCache, srcCacheCount, srcCaches);
+}
+
 ANGLE_INLINE VkResult PipelineCache::getCacheData(VkDevice device,
                                                   size_t *cacheSize,
                                                   void *cacheData)

src/libGLESv2/entry_points_egl_ext.cpp

@@ -1110,8 +1110,10 @@ ANGLE_EXPORT void EGLAPIENTRY EGL_SetBlobCacheFuncsANDROID(EGLDisplay dpy,
     ANGLE_EGL_TRY(thread, ValidateSetBlobCacheANDROID(display, set, get),
                   "eglSetBlobCacheFuncsANDROID", GetDisplayIfValid(display));
 
+    ANGLE_EGL_TRY(thread, display->setBlobCacheFuncs(set, get), "eglSetBlobCacheFuncsANDROID",
+                  GetDisplayIfValid(display));
+
     thread->setSuccess();
-    display->setBlobCacheFuncs(set, get);
 }
 
 EGLint EGLAPIENTRY EGL_ProgramCacheGetAttribANGLE(EGLDisplay dpy, EGLenum attrib)