Add a compiler option to prune unused function and prototypes

include/GLSLANG/ShaderLang.h

@@ -177,6 +177,11 @@ typedef enum {
   // It is intended as a workaround for drivers that do not handle
   // struct scopes correctly, including all Mac drivers and Linux AMD.
   SH_REGENERATE_STRUCT_NAMES = 0x80000,
+
+  // This flag makes the compiler not prune unused function early in the
+  // compilation process. Pruning coupled with SH_LIMIT_CALL_STACK_DEPTH
+  // helps avoid bad shaders causing stack overflows.
+  SH_DONT_PRUNE_UNUSED_FUNCTIONS = 0x100000,
 } ShCompileOptions;
 
 // Defines alternate strategies for implementing array index clamping.

src/compiler/translator/Compiler.cpp

@@ -246,6 +246,9 @@ TIntermNode *TCompiler::compileTreeImpl(const char* const shaderStrings[],
             success = tagUsedFunctions();
         }
 
+        if (success && !(compileOptions & SH_DONT_PRUNE_UNUSED_FUNCTIONS))
+            success = pruneUnusedFunctions(root);
+
         if (success && shaderVersion == 300 && shaderType == GL_FRAGMENT_SHADER)
             success = validateOutputs(root);
 
@@ -572,6 +575,59 @@ void TCompiler::internalTagUsedFunction(size_t index)
     }
 }
 
+// A predicate for the stl that returns if a top-level node is unused
+class TCompiler::UnusedPredicate
+{
+  public:
+    UnusedPredicate(const CallDAG *callDag, const std::vector<FunctionMetadata> *metadatas)
+        : mCallDag(callDag),
+          mMetadatas(metadatas)
+    {
+    }
+
+    bool operator ()(TIntermNode *node)
+    {
+        const TIntermAggregate *asAggregate = node->getAsAggregate();
+
+        if (asAggregate == nullptr)
+        {
+            return false;
+        }
+
+        if (!(asAggregate->getOp() == EOpFunction || asAggregate->getOp() == EOpPrototype))
+        {
+            return false;
+        }
+
+        size_t callDagIndex = mCallDag->findIndex(asAggregate);
+        if (callDagIndex == CallDAG::InvalidIndex)
+        {
+            // This happens only for unimplemented prototypes which are thus unused
+            ASSERT(asAggregate->getOp() == EOpPrototype);
+            return true;
+        }
+
+        ASSERT(callDagIndex < mMetadatas->size());
+        return !(*mMetadatas)[callDagIndex].used;
+    }
+
+  private:
+    const CallDAG *mCallDag;
+    const std::vector<FunctionMetadata> *mMetadatas;
+};
+
+bool TCompiler::pruneUnusedFunctions(TIntermNode *root)
+{
+    TIntermAggregate *rootNode = root->getAsAggregate();
+    ASSERT(rootNode != nullptr);
+
+    UnusedPredicate isUnused(&mCallDag, &functionMetadata);
+    TIntermSequence *sequence = rootNode->getSequence();
+    sequence->erase(std::remove_if(sequence->begin(), sequence->end(), isUnused), sequence->end());
+
+    return true;
+}
+
 bool TCompiler::validateOutputs(TIntermNode* root)
 {
     ValidateOutputs validateOutputs(infoSink.info, compileResources.MaxDrawBuffers);

src/compiler/translator/Compiler.h

@@ -165,6 +165,10 @@ class TCompiler : public TShHandleBase
     bool tagUsedFunctions();
     void internalTagUsedFunction(size_t index);
 
+    // Removes unused function declarations and prototypes from the AST
+    class UnusedPredicate;
+    bool pruneUnusedFunctions(TIntermNode *root);
+
     TIntermNode *compileTreeImpl(const char* const shaderStrings[],
         size_t numStrings, int compileOptions);
 

src/tests/angle_unittests.gypi

@@ -30,8 +30,9 @@
             '<(angle_path)/src/tests/compiler_tests/DebugShaderPrecision_test.cpp',
             '<(angle_path)/src/tests/compiler_tests/ExpressionLimit_test.cpp',
             '<(angle_path)/src/tests/compiler_tests/MalformedShader_test.cpp',
-            '<(angle_path)/src/tests/compiler_tests/ShaderExtension_test.cpp',
             '<(angle_path)/src/tests/compiler_tests/NV_draw_buffers_test.cpp',
+            '<(angle_path)/src/tests/compiler_tests/PruneUnusedFunctions_test.cpp',
+            '<(angle_path)/src/tests/compiler_tests/ShaderExtension_test.cpp',
             '<(angle_path)/src/tests/compiler_tests/ShaderVariable_test.cpp',
             '<(angle_path)/src/tests/compiler_tests/TypeTracking_test.cpp',
             '<(angle_path)/src/tests/preprocessor_tests/char_test.cpp',

src/tests/compiler_tests/PruneUnusedFunctions_test.cpp

+//
+// Copyright (c) 2015 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// PruneUnusedFunctions_test.cpp:
+//   Test for the pruning of unused function with the SH_PRUNE_UNUSED_FUNCTIONS compile flag
+//
+
+#include "angle_gl.h"
+#include "gtest/gtest.h"
+#include "GLSLANG/ShaderLang.h"
+#include "compiler/translator/TranslatorESSL.h"
+
+namespace
+{
+
+class PruneUnusedFunctionsTest : public testing::Test
+{
+  public:
+    PruneUnusedFunctionsTest() {}
+
+  protected:
+    void SetUp() override
+    {
+        ShBuiltInResources resources;
+        ShInitBuiltInResources(&resources);
+        resources.FragmentPrecisionHigh = 1;
+
+        mTranslator = new TranslatorESSL(GL_FRAGMENT_SHADER, SH_GLES3_SPEC);
+        ASSERT_TRUE(mTranslator->Init(resources));
+    }
+
+    void TearDown() override
+    {
+        SafeDelete(mTranslator);
+    }
+
+    void compile(const std::string &shaderString, bool prune)
+    {
+        const char *shaderStrings[] = { shaderString.c_str() };
+        int compilationFlags = SH_VARIABLES | SH_OBJECT_CODE | (prune ? 0 : SH_DONT_PRUNE_UNUSED_FUNCTIONS);
+        bool compilationSuccess = mTranslator->compile(shaderStrings, 1, compilationFlags);
+        TInfoSink &infoSink = mTranslator->getInfoSink();
+        if (!compilationSuccess)
+        {
+            FAIL() << "Shader compilation failed " << infoSink.info.str();
+        }
+        mTranslatedSource = infoSink.obj.str();
+    }
+
+    bool kept(const char *functionName, int nOccurences) const
+    {
+        size_t currentPos = 0;
+        while (nOccurences-- > 0)
+        {
+            auto position = mTranslatedSource.find(functionName, currentPos);
+            if (position == std::string::npos)
+            {
+                return false;
+            }
+            currentPos = position + 1;
+        }
+        return mTranslatedSource.find(functionName, currentPos) == std::string::npos;
+    }
+
+    bool removed(const char* functionName) const
+    {
+        return mTranslatedSource.find(functionName) == std::string::npos;
+    }
+
+  private:
+    TranslatorESSL *mTranslator;
+    std::string mTranslatedSource;
+};
+
+// Check that unused function and prototypes are removed iff the options is set
+TEST_F(PruneUnusedFunctionsTest, UnusedFunctionAndProto)
+{
+    const std::string &shaderString =
+        "precision mediump float;\n"
+        "float unused(float a);\n"
+        "void main() {\n"
+        "    gl_FragColor = vec4(1.0);\n"
+        "}\n"
+        "float unused(float a) {\n"
+        "    return a;\n"
+        "}\n";
+    compile(shaderString, true);
+    EXPECT_TRUE(removed("unused("));
+    EXPECT_TRUE(kept("main(", 1));
+
+    compile(shaderString, false);
+    EXPECT_TRUE(kept("unused(", 2));
+    EXPECT_TRUE(kept("main(", 1));
+}
+
+// Check that unimplemented prototypes are removed iff the options is set
+TEST_F(PruneUnusedFunctionsTest, UnimplementedPrototype)
+{
+    const std::string &shaderString =
+        "precision mediump float;\n"
+        "float unused(float a);\n"
+        "void main() {\n"
+        "    gl_FragColor = vec4(1.0);\n"
+        "}\n";
+    compile(shaderString, true);
+    EXPECT_TRUE(removed("unused("));
+    EXPECT_TRUE(kept("main(", 1));
+
+    compile(shaderString, false);
+    EXPECT_TRUE(kept("unused(", 1));
+    EXPECT_TRUE(kept("main(", 1));
+}
+
+// Check that used functions are not prunued (duh)
+TEST_F(PruneUnusedFunctionsTest, UsedFunction)
+{
+    const std::string &shaderString =
+        "precision mediump float;\n"
+        "float used(float a);\n"
+        "void main() {\n"
+        "    gl_FragColor = vec4(used(1.0));\n"
+        "}\n"
+        "float used(float a) {\n"
+        "    return a;\n"
+        "}\n";
+    compile(shaderString, true);
+    EXPECT_TRUE(kept("used(", 3));
+    EXPECT_TRUE(kept("main(", 1));
+
+    compile(shaderString, false);
+    EXPECT_TRUE(kept("used(", 3));
+    EXPECT_TRUE(kept("main(", 1));
+}
+
+}