Compress Program binaries saved in blob cache

BUILD.gn

@@ -594,6 +594,7 @@ angle_source_set("libANGLE_base") {
     ":translator",
   ]
   deps = [
+    ":angle_compression",
     ":angle_image_util",
     ":includes",
   ]

scripts/export_targets.py

@@ -223,6 +223,7 @@ IGNORED_DIRECTORIES = {
     '//third_party/vulkan-loader',
     '//third_party/vulkan-tools',
     '//third_party/vulkan-validation-layers',
+    '//third_party/zlib',
 }
 
 def has_all_includes(target_name: str, descs: dict) -> bool:

src/libANGLE/BlobCache.cpp

@@ -69,7 +69,8 @@ void BlobCache::populate(const BlobCache::Key &key, angle::MemoryBuffer &&value,
 
 bool BlobCache::get(angle::ScratchBuffer *scratchBuffer,
                     const BlobCache::Key &key,
-                    BlobCache::Value *valueOut)
+                    BlobCache::Value *valueOut,
+                    size_t *bufferSizeOut)
 {
     // Look into the application's cache, if there is such a cache
     if (areBlobCacheFuncsSet())
@@ -103,6 +104,7 @@ bool BlobCache::get(angle::ScratchBuffer *scratchBuffer,
         }
 
         *valueOut      = BlobCache::Value(scratchMemory->data(), scratchMemory->size());
+        *bufferSizeOut = valueSize;
         return true;
     }
 
@@ -124,6 +126,7 @@ bool BlobCache::get(angle::ScratchBuffer *scratchBuffer,
         }
 
         *valueOut      = BlobCache::Value(entry->first.data(), entry->first.size());
+        *bufferSizeOut = entry->first.size();
     }
     else
     {

src/libANGLE/BlobCache.h

@@ -102,7 +102,8 @@ class BlobCache final : angle::NonCopyable
     // set, those will be used.  Otherwise they key is looked up in this object's cache.
     ANGLE_NO_DISCARD bool get(angle::ScratchBuffer *scratchBuffer,
                               const BlobCache::Key &key,
-                              BlobCache::Value *valueOut);
+                              BlobCache::Value *valueOut,
+                              size_t *bufferSizeOut);
 
     // For querying the contents of the cache.
     ANGLE_NO_DISCARD bool getAt(size_t index,

src/libANGLE/BlobCache_unittest.cpp

@@ -58,7 +58,8 @@ TEST(BlobCacheTest, MaxSizedValue)
     EXPECT_FALSE(blobCache.empty());
 
     Blob blob;
-    EXPECT_FALSE(blobCache.get(nullptr, MakeKey(0), &blob));
+    size_t blobSize;
+    EXPECT_FALSE(blobCache.get(nullptr, MakeKey(0), &blob, &blobSize));
 
     blobCache.clear();
     EXPECT_TRUE(blobCache.empty());
@@ -75,7 +76,8 @@ TEST(BlobCacheTest, ManySmallValues)
         blobCache.populate(MakeKey(value), MakeBlob(1, value));
 
         Blob qvalue;
-        EXPECT_TRUE(blobCache.get(nullptr, MakeKey(value), &qvalue));
+        size_t blobSize;
+        EXPECT_TRUE(blobCache.get(nullptr, MakeKey(value), &qvalue, &blobSize));
         if (qvalue.size() > 0)
         {
             EXPECT_EQ(value, qvalue[0]);
@@ -89,7 +91,8 @@ TEST(BlobCacheTest, ManySmallValues)
     blobCache.populate(MakeKey(kSize), MakeBlob(1, kSize));
 
     Blob qvalue;
-    EXPECT_FALSE(blobCache.get(nullptr, MakeKey(0), &qvalue));
+    size_t blobSize;
+    EXPECT_FALSE(blobCache.get(nullptr, MakeKey(0), &qvalue, &blobSize));
 
     // Putting one large element cleans out the whole stack.
     blobCache.populate(MakeKey(kSize + 1), MakeBlob(kSize, kSize + 1));
@@ -98,9 +101,9 @@ TEST(BlobCacheTest, ManySmallValues)
 
     for (size_t value = 0; value <= kSize; ++value)
     {
-        EXPECT_FALSE(blobCache.get(nullptr, MakeKey(value), &qvalue));
+        EXPECT_FALSE(blobCache.get(nullptr, MakeKey(value), &qvalue, &blobSize));
     }
-    EXPECT_TRUE(blobCache.get(nullptr, MakeKey(kSize + 1), &qvalue));
+    EXPECT_TRUE(blobCache.get(nullptr, MakeKey(kSize + 1), &qvalue, &blobSize));
     if (qvalue.size() > 0)
     {
         EXPECT_EQ(kSize + 1, qvalue[0]);
@@ -124,7 +127,8 @@ TEST(BlobCacheTest, OversizeValue)
     blobCache.populate(MakeKey(5), MakeBlob(100));
 
     Blob qvalue;
-    EXPECT_FALSE(blobCache.get(nullptr, MakeKey(5), &qvalue));
+    size_t blobSize;
+    EXPECT_FALSE(blobCache.get(nullptr, MakeKey(5), &qvalue, &blobSize));
 }
 
 }  // namespace egl

src/libANGLE/MemoryProgramCache.cpp

@@ -21,6 +21,9 @@
 #include "libANGLE/renderer/ProgramImpl.h"
 #include "platform/PlatformMethods.h"
 
+#define USE_SYSTEM_ZLIB
+#include "compression_utils_portable.h"
+
 namespace gl
 {
 
@@ -140,11 +143,27 @@ angle::Result MemoryProgramCache::getProgram(const Context *context,
 
     ComputeHash(context, program, hashOut);
     egl::BlobCache::Value binaryProgram;
-    if (get(context, *hashOut, &binaryProgram))
+    size_t programSize = 0;
+    if (get(context, *hashOut, &binaryProgram, &programSize))
+    {
+        uint32_t uncompressedSize =
+            zlib_internal::GetGzipUncompressedSize(binaryProgram.data(), programSize);
+
+        std::vector<uint8_t> uncompressedData(uncompressedSize);
+        uLong destLen = uncompressedSize;
+        int zResult   = zlib_internal::GzipUncompressHelper(uncompressedData.data(), &destLen,
+                                                          binaryProgram.data(),
+                                                          static_cast<uLong>(programSize));
+
+        if (zResult != Z_OK)
         {
+            ERR() << "Failure to decompressed binary data: " << zResult << "\n";
+            return angle::Result::Incomplete;
+        }
+
         angle::Result result =
-            program->loadBinary(context, GL_PROGRAM_BINARY_ANGLE, binaryProgram.data(),
-                                static_cast<int>(binaryProgram.size()));
+            program->loadBinary(context, GL_PROGRAM_BINARY_ANGLE, uncompressedData.data(),
+                                static_cast<int>(uncompressedData.size()));
         ANGLE_HISTOGRAM_BOOLEAN("GPU.ANGLE.ProgramCache.LoadBinarySuccess",
                                 result == angle::Result::Continue);
         ANGLE_TRY(result);
@@ -170,9 +189,10 @@ angle::Result MemoryProgramCache::getProgram(const Context *context,
 
 bool MemoryProgramCache::get(const Context *context,
                              const egl::BlobCache::Key &programHash,
-                             egl::BlobCache::Value *programOut)
+                             egl::BlobCache::Value *programOut,
+                             size_t *programSizeOut)
 {
-    return mBlobCache.get(context->getScratchBuffer(), programHash, programOut);
+    return mBlobCache.get(context->getScratchBuffer(), programHash, programOut, programSizeOut);
 }
 
 bool MemoryProgramCache::getAt(size_t index,
@@ -200,16 +220,45 @@ angle::Result MemoryProgramCache::putProgram(const egl::BlobCache::Key &programH
     angle::MemoryBuffer serializedProgram;
     ANGLE_TRY(program->serialize(context, &serializedProgram));
 
+    // Compress the program data
+    uLong uncompressedSize       = static_cast<uLong>(serializedProgram.size());
+    uLong expectedCompressedSize = zlib_internal::GzipExpectedCompressedSize(uncompressedSize);
+
+    angle::MemoryBuffer compressedData;
+    if (!compressedData.resize(expectedCompressedSize))
+    {
+        ERR() << "Failed to allocate enough memory to hold compressed program. ("
+              << expectedCompressedSize << " bytes )";
+        return angle::Result::Incomplete;
+    }
+
+    int zResult = zlib_internal::GzipCompressHelper(compressedData.data(), &expectedCompressedSize,
+                                                    serializedProgram.data(), uncompressedSize,
+                                                    nullptr, nullptr);
+
+    if (zResult != Z_OK)
+    {
+        FATAL() << "Error compressing binary data: " << zResult;
+        return angle::Result::Incomplete;
+    }
+
+    // Resize the buffer to the actual compressed size
+    if (!compressedData.resize(expectedCompressedSize))
+    {
+        ERR() << "Failed to resize to actual compressed program size. (" << expectedCompressedSize
+              << " bytes )";
+        return angle::Result::Incomplete;
+    }
+
     ANGLE_HISTOGRAM_COUNTS("GPU.ANGLE.ProgramCache.ProgramBinarySizeBytes",
-                           static_cast<int>(serializedProgram.size()));
+                           static_cast<int>(compressedData.size()));
 
     // TODO(syoussefi): to be removed.  Compatibility for Chrome until it supports
     // EGL_ANDROID_blob_cache. http://anglebug.com/2516
     auto *platform = ANGLEPlatformCurrent();
-    platform->cacheProgram(platform, programHash, serializedProgram.size(),
-                           serializedProgram.data());
+    platform->cacheProgram(platform, programHash, compressedData.size(), compressedData.data());
 
-    mBlobCache.put(programHash, std::move(serializedProgram));
+    mBlobCache.put(programHash, std::move(compressedData));
     return angle::Result::Continue;
 }
 

src/libANGLE/MemoryProgramCache.h

@@ -35,7 +35,8 @@ class MemoryProgramCache final : angle::NonCopyable
     // Check if the cache contains a binary matching the specified program.
     bool get(const Context *context,
              const egl::BlobCache::Key &programHash,
-             egl::BlobCache::Value *programOut);
+             egl::BlobCache::Value *programOut,
+             size_t *programSizeOut);
 
     // For querying the contents of the cache.
     bool getAt(size_t index,

src/libANGLE/renderer/vulkan/RendererVk.cpp

@@ -1755,14 +1755,15 @@ angle::Result RendererVk::initPipelineCache(DisplayVk *display,
     initPipelineCacheVkKey();
 
     egl::BlobCache::Value initialData;
+    size_t dataSize = 0;
     *success = display->getBlobCache()->get(display->getScratchBuffer(), mPipelineCacheVkBlobKey,
-                                            &initialData);
+                                            &initialData, &dataSize);
 
     VkPipelineCacheCreateInfo pipelineCacheCreateInfo = {};
 
     pipelineCacheCreateInfo.sType           = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
     pipelineCacheCreateInfo.flags           = 0;
-    pipelineCacheCreateInfo.initialDataSize = *success ? initialData.size() : 0;
+    pipelineCacheCreateInfo.initialDataSize = *success ? dataSize : 0;
     pipelineCacheCreateInfo.pInitialData    = *success ? initialData.data() : nullptr;
 
     ANGLE_VK_TRY(display, pipelineCache->init(mDevice, pipelineCacheCreateInfo));