Prevent stack overflow in macro expansion

src/compiler/preprocessor/DiagnosticsBase.cpp

@@ -82,6 +82,8 @@ std::string Diagnostics::message(ID id)
         return "Too many arguments for macro";
       case PP_MACRO_DUPLICATE_PARAMETER_NAMES:
         return "duplicate macro parameter name";
+      case PP_MACRO_INVOCATION_CHAIN_TOO_DEEP:
+          return "macro invocation chain too deep";
       case PP_CONDITIONAL_ENDIF_WITHOUT_IF:
         return "unexpected #endif found without a matching #if";
       case PP_CONDITIONAL_ELSE_WITHOUT_IF:

src/compiler/preprocessor/DiagnosticsBase.h

@@ -48,6 +48,7 @@ class Diagnostics
         PP_MACRO_TOO_FEW_ARGS,
         PP_MACRO_TOO_MANY_ARGS,
         PP_MACRO_DUPLICATE_PARAMETER_NAMES,
+        PP_MACRO_INVOCATION_CHAIN_TOO_DEEP,
         PP_CONDITIONAL_ENDIF_WITHOUT_IF,
         PP_CONDITIONAL_ELSE_WITHOUT_IF,
         PP_CONDITIONAL_ELSE_AFTER_ELSE,

src/compiler/preprocessor/DirectiveParser.cpp

@@ -202,14 +202,16 @@ class DefinedParser : public Lexer
 DirectiveParser::DirectiveParser(Tokenizer *tokenizer,
                                  MacroSet *macroSet,
                                  Diagnostics *diagnostics,
-                                 DirectiveHandler *directiveHandler)
+                                 DirectiveHandler *directiveHandler,
+                                 int maxMacroExpansionDepth)
     : mPastFirstStatement(false),
       mSeenNonPreprocessorToken(false),
       mTokenizer(tokenizer),
       mMacroSet(macroSet),
       mDiagnostics(diagnostics),
       mDirectiveHandler(directiveHandler),
-      mShaderVersion(100)
+      mShaderVersion(100),
+      mMaxMacroExpansionDepth(maxMacroExpansionDepth)
 {
 }
 
@@ -843,7 +845,7 @@ void DirectiveParser::parseLine(Token *token)
     bool parsedFileNumber = false;
     int line = 0, file = 0;
 
-    MacroExpander macroExpander(mTokenizer, mMacroSet, mDiagnostics);
+    MacroExpander macroExpander(mTokenizer, mMacroSet, mDiagnostics, mMaxMacroExpansionDepth);
 
     // Lex the first token after "#line" so we can check it for EOD.
     macroExpander.lex(token);
@@ -952,7 +954,7 @@ int DirectiveParser::parseExpressionIf(Token *token)
     ASSERT((getDirective(token) == DIRECTIVE_IF) || (getDirective(token) == DIRECTIVE_ELIF));
 
     DefinedParser definedParser(mTokenizer, mMacroSet, mDiagnostics);
-    MacroExpander macroExpander(&definedParser, mMacroSet, mDiagnostics);
+    MacroExpander macroExpander(&definedParser, mMacroSet, mDiagnostics, mMaxMacroExpansionDepth);
     ExpressionParser expressionParser(&macroExpander, mDiagnostics);
 
     int expression = 0;

src/compiler/preprocessor/DirectiveParser.h

@@ -24,7 +24,8 @@ class DirectiveParser : public Lexer
     DirectiveParser(Tokenizer *tokenizer,
                     MacroSet *macroSet,
                     Diagnostics *diagnostics,
-                    DirectiveHandler *directiveHandler);
+                    DirectiveHandler *directiveHandler,
+                    int maxMacroExpansionDepth);
 
     void lex(Token *token) override;
 
@@ -76,6 +77,7 @@ class DirectiveParser : public Lexer
     Diagnostics *mDiagnostics;
     DirectiveHandler *mDirectiveHandler;
     int mShaderVersion;
+    int mMaxMacroExpansionDepth;
 };
 
 }  // namespace pp

src/compiler/preprocessor/MacroExpander.cpp

@@ -77,11 +77,15 @@ MacroExpander::ScopedMacroReenabler::~ScopedMacroReenabler()
     mExpander->mMacrosToReenable.clear();
 }
 
-MacroExpander::MacroExpander(Lexer *lexer, MacroSet *macroSet, Diagnostics *diagnostics)
+MacroExpander::MacroExpander(Lexer *lexer,
+                             MacroSet *macroSet,
+                             Diagnostics *diagnostics,
+                             int allowedMacroExpansionDepth)
     : mLexer(lexer),
       mMacroSet(macroSet),
       mDiagnostics(diagnostics),
       mTotalTokensInContexts(0),
+      mAllowedMacroExpansionDepth(allowedMacroExpansionDepth),
       mDeferReenablingMacros(false)
 {
 }
@@ -377,7 +381,13 @@ bool MacroExpander::collectMacroArgs(const Macro &macro,
     for (auto &arg : *args)
     {
         TokenLexer lexer(&arg);
-        MacroExpander expander(&lexer, mMacroSet, mDiagnostics);
+        if (mAllowedMacroExpansionDepth < 1)
+        {
+            mDiagnostics->report(Diagnostics::PP_MACRO_INVOCATION_CHAIN_TOO_DEEP, token.location,
+                                 token.text);
+            return false;
+        }
+        MacroExpander expander(&lexer, mMacroSet, mDiagnostics, mAllowedMacroExpansionDepth - 1);
 
         arg.clear();
         expander.lex(&token);

src/compiler/preprocessor/MacroExpander.h

@@ -22,7 +22,10 @@ struct SourceLocation;
 class MacroExpander : public Lexer
 {
   public:
-    MacroExpander(Lexer *lexer, MacroSet *macroSet, Diagnostics *diagnostics);
+    MacroExpander(Lexer *lexer,
+                  MacroSet *macroSet,
+                  Diagnostics *diagnostics,
+                  int allowedMacroExpansionDepth);
     ~MacroExpander() override;
 
     void lex(Token *token) override;
@@ -68,6 +71,8 @@ class MacroExpander : public Lexer
     std::vector<MacroContext *> mContextStack;
     size_t mTotalTokensInContexts;
 
+    int mAllowedMacroExpansionDepth;
+
     bool mDeferReenablingMacros;
     std::vector<const Macro *> mMacrosToReenable;
 

src/compiler/preprocessor/Preprocessor.cpp

@@ -25,19 +25,26 @@ struct PreprocessorImpl
     DirectiveParser directiveParser;
     MacroExpander macroExpander;
 
-    PreprocessorImpl(Diagnostics *diag, DirectiveHandler *directiveHandler)
+    PreprocessorImpl(Diagnostics *diag,
+                     DirectiveHandler *directiveHandler,
+                     const PreprocessorSettings &settings)
         : diagnostics(diag),
           tokenizer(diag),
-          directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
-          macroExpander(&directiveParser, &macroSet, diag)
+          directiveParser(&tokenizer,
+                          &macroSet,
+                          diag,
+                          directiveHandler,
+                          settings.maxMacroExpansionDepth),
+          macroExpander(&directiveParser, &macroSet, diag, settings.maxMacroExpansionDepth)
     {
     }
 };
 
 Preprocessor::Preprocessor(Diagnostics *diagnostics,
-                           DirectiveHandler *directiveHandler)
+                           DirectiveHandler *directiveHandler,
+                           const PreprocessorSettings &settings)
 {
-    mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
+    mImpl = new PreprocessorImpl(diagnostics, directiveHandler, settings);
 }
 
 Preprocessor::~Preprocessor()

src/compiler/preprocessor/Preprocessor.h

@@ -19,10 +19,18 @@ class DirectiveHandler;
 struct PreprocessorImpl;
 struct Token;
 
+struct PreprocessorSettings : angle::NonCopyable
+{
+    PreprocessorSettings() : maxMacroExpansionDepth(1000) {}
+    int maxMacroExpansionDepth;
+};
+
 class Preprocessor : angle::NonCopyable
 {
   public:
-    Preprocessor(Diagnostics *diagnostics, DirectiveHandler *directiveHandler);
+    Preprocessor(Diagnostics *diagnostics,
+                 DirectiveHandler *directiveHandler,
+                 const PreprocessorSettings &settings);
     ~Preprocessor();
 
     // count: specifies the number of elements in the string and length arrays.

src/compiler/translator/ParseContext.cpp

@@ -99,7 +99,7 @@ TParseContext::TParseContext(TSymbolTable &symt,
                         mShaderVersion,
                         mShaderType,
                         resources.WEBGL_debug_shader_precision == 1),
-      mPreprocessor(&mDiagnostics, &mDirectiveHandler),
+      mPreprocessor(&mDiagnostics, &mDirectiveHandler, pp::PreprocessorSettings()),
       mScanner(nullptr),
       mUsesFragData(false),
       mUsesFragColor(false),

src/tests/preprocessor_tests/PreprocessorTest.cpp

@@ -7,23 +7,61 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-void PreprocessorTest::preprocess(const char* input, const char* expected)
+void SimplePreprocessorTest::preprocess(const char *input,
+                                        std::stringstream *output,
+                                        pp::Preprocessor *preprocessor)
 {
-    ASSERT_TRUE(mPreprocessor.init(1, &input, NULL));
+    ASSERT_TRUE(preprocessor->init(1, &input, NULL));
 
     int line = 1;
     pp::Token token;
-    std::stringstream stream;
     do
     {
-        mPreprocessor.lex(&token);
-        for (; line < token.location.line; ++line)
+        preprocessor->lex(&token);
+        if (output)
         {
-            stream << "\n";
+            for (; line < token.location.line; ++line)
+            {
+                *output << "\n";
+            }
+            *output << token;
         }
-        stream << token;
     } while (token.type != pp::Token::LAST);
+}
+
+void SimplePreprocessorTest::preprocess(const char *input, const pp::PreprocessorSettings &settings)
+{
+    pp::Preprocessor preprocessor(&mDiagnostics, &mDirectiveHandler, settings);
+    preprocess(input, nullptr, &preprocessor);
+}
+
+void SimplePreprocessorTest::preprocess(const char *input)
+{
+    preprocess(input, pp::PreprocessorSettings());
+}
 
-    std::string actual = stream.str();
+void SimplePreprocessorTest::preprocess(const char *input, const char *expected)
+{
+    pp::Preprocessor preprocessor(&mDiagnostics, &mDirectiveHandler, pp::PreprocessorSettings());
+    std::stringstream output;
+    preprocess(input, &output, &preprocessor);
+
+    std::string actual = output.str();
     EXPECT_STREQ(expected, actual.c_str());
 }
+
+void SimplePreprocessorTest::lexSingleToken(const char *input, pp::Token *token)
+{
+    pp::Preprocessor preprocessor(&mDiagnostics, &mDirectiveHandler, pp::PreprocessorSettings());
+    ASSERT_TRUE(preprocessor.init(1, &input, nullptr));
+    preprocessor.lex(token);
+}
+
+void SimplePreprocessorTest::lexSingleToken(size_t count,
+                                            const char *const input[],
+                                            pp::Token *token)
+{
+    pp::Preprocessor preprocessor(&mDiagnostics, &mDirectiveHandler, pp::PreprocessorSettings());
+    ASSERT_TRUE(preprocessor.init(count, input, nullptr));
+    preprocessor.lex(token);
+}
\ No newline at end of file

src/tests/preprocessor_tests/PreprocessorTest.h

@@ -16,15 +16,35 @@
 class PreprocessorTest : public testing::Test
 {
   protected:
-    PreprocessorTest() : mPreprocessor(&mDiagnostics, &mDirectiveHandler) { }
-
-    // Preprocesses the input string and verifies that it matches
-    // expected output.
-    void preprocess(const char* input, const char* expected);
+    PreprocessorTest()
+        : mPreprocessor(&mDiagnostics, &mDirectiveHandler, pp::PreprocessorSettings())
+    {
+    }
 
     MockDiagnostics mDiagnostics;
     MockDirectiveHandler mDirectiveHandler;
     pp::Preprocessor mPreprocessor;
 };
 
+class SimplePreprocessorTest : public testing::Test
+{
+  protected:
+    // Preprocesses the input string.
+    void preprocess(const char *input);
+    void preprocess(const char *input, const pp::PreprocessorSettings &settings);
+
+    // Preprocesses the input string and verifies that it matches expected output.
+    void preprocess(const char *input, const char *expected);
+
+    // Lexes a single token from input and writes it to token.
+    void lexSingleToken(const char *input, pp::Token *token);
+    void lexSingleToken(size_t count, const char *const input[], pp::Token *token);
+
+    MockDiagnostics mDiagnostics;
+    MockDirectiveHandler mDirectiveHandler;
+
+  private:
+    void preprocess(const char *input, std::stringstream *output, pp::Preprocessor *preprocessor);
+};
+
 #endif  // PREPROCESSOR_TESTS_PREPROCESSOR_TEST_H_

src/tests/preprocessor_tests/comment_test.cpp

@@ -7,8 +7,7 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-class CommentTest : public PreprocessorTest,
-                    public testing::WithParamInterface<const char*>
+class CommentTest : public SimplePreprocessorTest, public testing::WithParamInterface<const char *>
 {
 };
 
@@ -16,10 +15,8 @@ TEST_P(CommentTest, CommentIgnored)
 {
     const char* str = GetParam();
 
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
-
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(str, &token);
     EXPECT_EQ(pp::Token::LAST, token.type);
 }
 
@@ -38,7 +35,7 @@ INSTANTIATE_TEST_CASE_P(BlockComment, CommentTest,
                                         "/***/",     // With lone '*'.
                                         "/*\"*/"));  // Invalid character.
 
-class BlockCommentTest : public PreprocessorTest
+class BlockCommentTest : public SimplePreprocessorTest
 {
 };
 
@@ -46,10 +43,8 @@ TEST_F(BlockCommentTest, CommentReplacedWithSpace)
 {
     const char* str = "/*foo*/bar";
 
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
-
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(str, &token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_EQ("bar", token.text);
     EXPECT_TRUE(token.hasLeadingSpace());
@@ -59,11 +54,8 @@ TEST_F(BlockCommentTest, UnterminatedComment)
 {
     const char* str = "/*foo";
 
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
-
     using testing::_;
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_EOF_IN_COMMENT, _, _));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }

src/tests/preprocessor_tests/define_test.cpp

@@ -4,12 +4,14 @@
 // found in the LICENSE file.
 //
 
+#include <sstream>
+
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
 using testing::_;
 
-class DefineTest : public PreprocessorTest
+class DefineTest : public SimplePreprocessorTest
 {
 };
 
@@ -849,10 +851,9 @@ TEST_F(DefineTest, Predefined_VERSION)
 TEST_F(DefineTest, Predefined_LINE1)
 {
     const char* str = "\n\n__LINE__";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(str, &token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
     EXPECT_EQ("3", token.text);
 }
@@ -861,10 +862,9 @@ TEST_F(DefineTest, Predefined_LINE2)
 {
     const char* str = "#line 10\n"
                       "__LINE__\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(str, &token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
     EXPECT_EQ("10", token.text);
 }
@@ -872,10 +872,9 @@ TEST_F(DefineTest, Predefined_LINE2)
 TEST_F(DefineTest, Predefined_FILE1)
 {
     const char* const str[] = {"", "", "__FILE__"};
-    ASSERT_TRUE(mPreprocessor.init(3, str, NULL));
 
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(3, str, &token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
     EXPECT_EQ("2", token.text);
 }
@@ -883,10 +882,9 @@ TEST_F(DefineTest, Predefined_FILE1)
 TEST_F(DefineTest, Predefined_FILE2)
 {
     const char* const str[] = {"#line 10 20\n", "__FILE__"};
-    ASSERT_TRUE(mPreprocessor.init(2, str, NULL));
 
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(2, str, &token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
     EXPECT_EQ("21", token.text);
 }
@@ -986,3 +984,27 @@ TEST_F(DefineTest, RecursiveMacroNameInsideIncompleteMacroInvocationInMacroExpan
         "\n";
     preprocess(input, expected);
 }
+
+// The macro invocations form a long chain. The macro expander should protect against stack overflow
+// and generate an error in this case.
+TEST_F(DefineTest, LongMacroInvocationChain)
+{
+    std::stringstream inputStream;
+    std::stringstream expectedStream;
+
+    inputStream << "#define b(x) x\n";
+    inputStream << "#define a0(x) foo x\n";
+    for (int i = 1; i < 20; ++i)
+    {
+        inputStream << "#define a" << i << "(x) b(a" << (i - 1) << "(x))\n";
+    }
+    inputStream << "a19(y)\n";
+
+    EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_MACRO_INVOCATION_CHAIN_TOO_DEEP,
+                                    pp::SourceLocation(0, 22), _));
+
+    pp::PreprocessorSettings settings;
+    settings.maxMacroExpansionDepth = 19;
+
+    preprocess(inputStream.str().c_str(), settings);
+}

src/tests/preprocessor_tests/error_test.cpp

@@ -7,7 +7,7 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-class ErrorTest : public PreprocessorTest
+class ErrorTest : public SimplePreprocessorTest
 {
 };
 

src/tests/preprocessor_tests/extension_test.cpp

@@ -7,7 +7,7 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-class ExtensionTest : public PreprocessorTest
+class ExtensionTest : public SimplePreprocessorTest
 {
 };
 

src/tests/preprocessor_tests/identifier_test.cpp

@@ -9,16 +9,15 @@
 
 #define CLOSED_RANGE(x, y) testing::Range(x, static_cast<char>((y) + 1))
 
-class IdentifierTest : public PreprocessorTest
+class IdentifierTest : public SimplePreprocessorTest
 {
 protected:
     void expectIdentifier(const std::string& str)
     {
         const char* cstr = str.c_str();
-        ASSERT_TRUE(mPreprocessor.init(1, &cstr, 0));
 
         pp::Token token;
-        mPreprocessor.lex(&token);
+        lexSingleToken(cstr, &token);
         EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
         EXPECT_EQ(str, token.text);
     }

src/tests/preprocessor_tests/if_test.cpp

@@ -7,7 +7,7 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-class IfTest : public PreprocessorTest
+class IfTest : public SimplePreprocessorTest
 {
 };
 
@@ -609,100 +609,86 @@ TEST_F(IfTest, MissingExpression)
 {
     const char* str = "#if\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_INVALID_EXPRESSION,
                       pp::SourceLocation(0, 1),
                       "syntax error"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, DivisionByZero)
 {
     const char* str = "#if 1 / (3 - (1 + 2))\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_DIVISION_BY_ZERO,
                       pp::SourceLocation(0, 1), "1 / 0"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, ModuloByZero)
 {
     const char* str = "#if 1 % (3 - (1 + 2))\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_DIVISION_BY_ZERO,
                       pp::SourceLocation(0, 1), "1 % 0"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, DecIntegerOverflow)
 {
     const char* str = "#if 4294967296\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_INTEGER_OVERFLOW,
                       pp::SourceLocation(0, 1), "4294967296"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, OctIntegerOverflow)
 {
     const char* str = "#if 077777777777\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_INTEGER_OVERFLOW,
                       pp::SourceLocation(0, 1), "077777777777"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, HexIntegerOverflow)
 {
     const char* str = "#if 0xfffffffff\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_INTEGER_OVERFLOW,
                       pp::SourceLocation(0, 1), "0xfffffffff"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, UndefinedMacro)
 {
     const char* str = "#if UNDEFINED\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                       pp::SourceLocation(0, 1),
                       "UNDEFINED"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, InvalidExpressionIgnoredForExcludedElif)
@@ -728,43 +714,37 @@ TEST_F(IfTest, InvalidExpressionIgnoredForExcludedElif)
 TEST_F(IfTest, ElseWithoutIf)
 {
     const char* str = "#else\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_ELSE_WITHOUT_IF,
                       pp::SourceLocation(0, 1),
                       "else"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, ElifWithoutIf)
 {
     const char* str = "#elif 1\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_ELIF_WITHOUT_IF,
                       pp::SourceLocation(0, 1),
                       "elif"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, EndifWithoutIf)
 {
     const char* str = "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_ENDIF_WITHOUT_IF,
                       pp::SourceLocation(0, 1),
                       "endif"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, ElseAfterElse)
@@ -773,15 +753,13 @@ TEST_F(IfTest, ElseAfterElse)
                       "#else\n"
                       "#else\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_ELSE_AFTER_ELSE,
                       pp::SourceLocation(0, 3),
                       "else"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, ElifAfterElse)
@@ -790,43 +768,37 @@ TEST_F(IfTest, ElifAfterElse)
                       "#else\n"
                       "#elif 0\n"
                       "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_ELIF_AFTER_ELSE,
                       pp::SourceLocation(0, 3),
                       "elif"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, UnterminatedIf)
 {
     const char* str = "#if 1\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_UNTERMINATED,
                       pp::SourceLocation(0, 1),
                       "if"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 TEST_F(IfTest, UnterminatedIfdef)
 {
     const char* str = "#ifdef foo\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_CONDITIONAL_UNTERMINATED,
                       pp::SourceLocation(0, 1),
                       "ifdef"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // The preprocessor only allows one expression to follow an #if directive.
@@ -836,13 +808,11 @@ TEST_F(IfTest, ExtraIntExpression)
     const char *str =
         "#if 1 1\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 1), "1"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // The preprocessor only allows one expression to follow an #if directive.
@@ -853,13 +823,11 @@ TEST_F(IfTest, ExtraIdentifierExpression)
         "#define one 1\n"
         "#if 1 one\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "1"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Divide by zero that's not evaluated because of short-circuiting should not cause an error.
@@ -902,15 +870,13 @@ TEST_F(IfTest, DefinedOperatorValidAfterMacroExpansion)
         "#if !foo bar\n"
         "pass\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "defined"));
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "bar"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Defined operator produced by macro expansion has undefined behavior according to C++ spec,
@@ -922,15 +888,13 @@ TEST_F(IfTest, UnterminatedDefinedInMacro)
         "#define foo defined(\n"
         "#if foo\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "defined"));
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "("));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Defined operator produced by macro expansion has undefined behavior according to C++ spec,
@@ -942,15 +906,13 @@ TEST_F(IfTest, UnterminatedDefinedInMacro2)
         "#define foo defined(bar\n"
         "#if foo\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "defined"));
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN,
                                     pp::SourceLocation(0, 2), "("));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Undefined shift: negative shift offset.
@@ -960,13 +922,11 @@ TEST_F(IfTest, BitShiftLeftOperatorNegativeOffset)
         "#if 2 << -1 == 1\n"
         "foo\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_UNDEFINED_SHIFT, pp::SourceLocation(0, 1), "2 << -1"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Undefined shift: shift offset is out of range.
@@ -976,13 +936,11 @@ TEST_F(IfTest, BitShiftLeftOperatorOffset32)
         "#if 2 << 32 == 1\n"
         "foo\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_UNDEFINED_SHIFT, pp::SourceLocation(0, 1), "2 << 32"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Left hand side of shift is negative.
@@ -1007,13 +965,11 @@ TEST_F(IfTest, BitShiftRightOperatorNegativeOffset)
         "#if 2 >> -1 == 4\n"
         "foo\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_UNDEFINED_SHIFT, pp::SourceLocation(0, 1), "2 >> -1"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Undefined shift: shift offset is out of range.
@@ -1023,13 +979,11 @@ TEST_F(IfTest, BitShiftRightOperatorOffset32)
         "#if 2 >> 32 == 0\n"
         "foo\n"
         "#endif\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_UNDEFINED_SHIFT, pp::SourceLocation(0, 1), "2 >> 32"));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 // Left hand side of shift is negative.

src/tests/preprocessor_tests/number_test.cpp

@@ -9,21 +9,19 @@
 
 #define CLOSED_RANGE(x, y) testing::Range(x, static_cast<char>((y) + 1))
 
-class InvalidNumberTest : public PreprocessorTest,
-                          public testing::WithParamInterface<const char*>
+class InvalidNumberTest : public SimplePreprocessorTest,
+                          public testing::WithParamInterface<const char *>
 {
 };
 
 TEST_P(InvalidNumberTest, InvalidNumberIdentified)
 {
     const char* str = GetParam();
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
 
     using testing::_;
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_INVALID_NUMBER, _, str));
 
-    pp::Token token;
-    mPreprocessor.lex(&token);
+    preprocess(str);
 }
 
 INSTANTIATE_TEST_CASE_P(InvalidIntegers, InvalidNumberTest,
@@ -34,8 +32,7 @@ INSTANTIATE_TEST_CASE_P(InvalidFloats, InvalidNumberTest,
                         testing::Values("1eg", "0.a", "0.1.2", ".0a", ".0.1"));
 
 typedef std::tr1::tuple<const char*, char> IntegerParams;
-class IntegerTest : public PreprocessorTest,
-                    public testing::WithParamInterface<IntegerParams>
+class IntegerTest : public SimplePreprocessorTest, public testing::WithParamInterface<IntegerParams>
 {
 };
 
@@ -45,10 +42,8 @@ TEST_P(IntegerTest, Identified)
     str.push_back(std::tr1::get<1>(GetParam()));  // digit.
     const char* cstr = str.c_str();
 
-    ASSERT_TRUE(mPreprocessor.init(1, &cstr, 0));
-
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(cstr, &token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
     EXPECT_EQ(str, token.text);
 }
@@ -78,16 +73,15 @@ INSTANTIATE_TEST_CASE_P(HexadecimalInteger_A_F,
                         testing::Combine(testing::Values("0x", "0X"),
                                          CLOSED_RANGE('A', 'F')));
 
-class FloatTest : public PreprocessorTest
+class FloatTest : public SimplePreprocessorTest
 {
   protected:
     void expectFloat(const std::string& str)
     {
         const char* cstr = str.c_str();
-        ASSERT_TRUE(mPreprocessor.init(1, &cstr, 0));
 
         pp::Token token;
-        mPreprocessor.lex(&token);
+        lexSingleToken(cstr, &token);
         EXPECT_EQ(pp::Token::CONST_FLOAT, token.type);
         EXPECT_EQ(str, token.text);
     }

src/tests/preprocessor_tests/operator_test.cpp

@@ -13,7 +13,7 @@ struct OperatorTestParam
     int op;
 };
 
-class OperatorTest : public PreprocessorTest,
+class OperatorTest : public SimplePreprocessorTest,
                      public testing::WithParamInterface<OperatorTestParam>
 {
 };
@@ -22,10 +22,8 @@ TEST_P(OperatorTest, Identified)
 {
     OperatorTestParam param = GetParam();
 
-    ASSERT_TRUE(mPreprocessor.init(1, &param.str, 0));
-
     pp::Token token;
-    mPreprocessor.lex(&token);
+    lexSingleToken(param.str, &token);
     EXPECT_EQ(param.op, token.type);
     EXPECT_EQ(param.str, token.text);
 }

src/tests/preprocessor_tests/pragma_test.cpp

@@ -7,7 +7,7 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-class PragmaTest : public PreprocessorTest
+class PragmaTest : public SimplePreprocessorTest
 {
 };
 

src/tests/preprocessor_tests/version_test.cpp

@@ -7,7 +7,7 @@
 #include "PreprocessorTest.h"
 #include "compiler/preprocessor/Token.h"
 
-class VersionTest : public PreprocessorTest
+class VersionTest : public SimplePreprocessorTest
 {
 };
 
@@ -100,25 +100,19 @@ TEST_F(VersionTest, AfterValidToken)
 {
     const char* str = "foo\n"
                       "#version 200\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     using testing::_;
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT,
                       pp::SourceLocation(0, 2), _));
 
-    pp::Token token;
-    do
-    {
-        mPreprocessor.lex(&token);
-    } while (token.type != pp::Token::LAST);
+    preprocess(str);
 }
 
 TEST_F(VersionTest, AfterInvalidToken)
 {
     const char* str = "$\n"
                       "#version 200\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     using testing::_;
     EXPECT_CALL(mDiagnostics,
@@ -128,36 +122,26 @@ TEST_F(VersionTest, AfterInvalidToken)
                 print(pp::Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT,
                       pp::SourceLocation(0, 2), _));
 
-    pp::Token token;
-    do
-    {
-        mPreprocessor.lex(&token);
-    } while (token.type != pp::Token::LAST);
+    preprocess(str);
 }
 
 TEST_F(VersionTest, AfterValidDirective)
 {
     const char* str = "#\n"
                       "#version 200\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     using testing::_;
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT,
                       pp::SourceLocation(0, 2), _));
 
-    pp::Token token;
-    do
-    {
-        mPreprocessor.lex(&token);
-    } while (token.type != pp::Token::LAST);
+    preprocess(str);
 }
 
 TEST_F(VersionTest, AfterInvalidDirective)
 {
     const char* str = "#foo\n"
                       "#version 200\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     using testing::_;
     EXPECT_CALL(mDiagnostics,
@@ -167,11 +151,7 @@ TEST_F(VersionTest, AfterInvalidDirective)
                 print(pp::Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT,
                       pp::SourceLocation(0, 2), _));
 
-    pp::Token token;
-    do
-    {
-        mPreprocessor.lex(&token);
-    } while (token.type != pp::Token::LAST);
+    preprocess(str);
 }
 
 TEST_F(VersionTest, AfterExcludedBlock)
@@ -180,18 +160,13 @@ TEST_F(VersionTest, AfterExcludedBlock)
                       "foo\n"
                       "#endif\n"
                       "#version 200\n";
-    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
 
     using testing::_;
     EXPECT_CALL(mDiagnostics,
                 print(pp::Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT,
                       pp::SourceLocation(0, 4), _));
 
-    pp::Token token;
-    do
-    {
-        mPreprocessor.lex(&token);
-    } while (token.type != pp::Token::LAST);
+    preprocess(str);
 }
 
 struct VersionTestParam