Implemented macro expansion.

src/compiler/preprocessor/new/Diagnostics.h

@@ -33,6 +33,9 @@ class Diagnostics
         UNEXPECTED_TOKEN,
         MACRO_NAME_RESERVED,
         MACRO_REDEFINED,
+        MACRO_UNTERMINATED_INVOCATION,
+        MACRO_TOO_FEW_ARGS,
+        MACRO_TOO_MANY_ARGS,
         INVALID_EXTENSION_NAME,
         INVALID_EXTENSION_BEHAVIOR,
         INVALID_EXTENSION_DIRECTIVE,

src/compiler/preprocessor/new/DirectiveParser.cpp

@@ -120,11 +120,6 @@ void DirectiveParser::parseDirective(Token* token)
             parseLine(token);
     }
 
-    if ((token->type != '\n') && (token->type != 0))
-        mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN,
-                             token->location,
-                             token->value);
-
     while (token->type != '\n')
     {
         if (token->type == 0) {
@@ -172,7 +167,7 @@ void DirectiveParser::parseDefine(Token* token)
                 break;
             macro.parameters.push_back(token->value);
 
-            mTokenizer->lex(token);  // Get comma.
+            mTokenizer->lex(token);  // Get ','.
         } while (token->type == ',');
 
         if (token->type != ')')
@@ -182,6 +177,7 @@ void DirectiveParser::parseDefine(Token* token)
                                  token->value);
             return;
         }
+        mTokenizer->lex(token);  // Get ')'.
     }
 
     while ((token->type != '\n') && (token->type != Token::LAST))
@@ -193,6 +189,12 @@ void DirectiveParser::parseDefine(Token* token)
         macro.replacements.push_back(*token);
         mTokenizer->lex(token);
     }
+    if (!macro.replacements.empty())
+    {
+        // Whitespace preceding the replacement list is not considered part of
+        // the replacement list for either form of macro.
+        macro.replacements.front().setHasLeadingSpace(false);
+    }
 
     // Check for macro redefinition.
     MacroSet::const_iterator iter = mMacroSet->find(macro.name);

src/compiler/preprocessor/new/Macro.h

@@ -23,13 +23,17 @@ struct Macro
         kTypeObj,
         kTypeFunc
     };
+    typedef std::vector<std::string> Parameters;
+    typedef std::vector<Token> Replacements;
 
+    Macro() : disabled(false), type(kTypeObj) { }
     bool equals(const Macro& other) const;
 
+    mutable bool disabled;
     Type type;
     std::string name;
-    std::vector<std::string> parameters;
-    std::vector<Token> replacements;
+    Parameters parameters;
+    Replacements replacements;
 };
 
 typedef std::map<std::string, Macro> MacroSet;

src/compiler/preprocessor/new/MacroExpander.cpp

@@ -6,9 +6,46 @@
 
 #include "MacroExpander.h"
 
+#include <algorithm>
+#include <cassert>
+
+#include "Diagnostics.h"
+#include "Token.h"
+
 namespace pp
 {
 
+class TokenLexer : public Lexer
+{
+ public:
+    typedef std::vector<Token> TokenVector;
+
+    TokenLexer(TokenVector* tokens)
+    {
+        tokens->swap(mTokens);
+        mIter = mTokens.begin();
+    }
+
+    virtual void lex(Token* token)
+    {
+        if (mIter == mTokens.end())
+        {
+            token->reset();
+            token->type = Token::LAST;
+        }
+        else
+        {
+            *token = *mIter++;
+        }
+    }
+
+ private:
+    PP_DISALLOW_COPY_AND_ASSIGN(TokenLexer);
+
+    TokenVector mTokens;
+    TokenVector::const_iterator mIter;
+};
+
 MacroExpander::MacroExpander(Lexer* lexer,
                              MacroSet* macroSet,
                              Diagnostics* diagnostics) :
@@ -18,10 +55,292 @@ MacroExpander::MacroExpander(Lexer* lexer,
 {
 }
 
+MacroExpander::~MacroExpander()
+{
+    assert(!mReserveToken.get());
+    assert(mContextStack.empty());
+}
+
 void MacroExpander::lex(Token* token)
 {
-    // TODO(alokp): Implement me.
-    mLexer->lex(token);
+    while (true)
+    {
+        getToken(token);
+
+        if (token->type != Token::IDENTIFIER)
+            break;
+
+        if (token->expansionDisabled())
+            break;
+
+        MacroSet::const_iterator iter = mMacroSet->find(token->value);
+        if (iter == mMacroSet->end())
+            break;
+
+        const Macro& macro = iter->second;
+        if (macro.disabled)
+        {
+            // If a particular token is not expanded, it is never expanded.
+            token->setExpansionDisabled(true);
+            break;
+        }
+        if ((macro.type == Macro::kTypeFunc) && !isNextTokenLeftParen())
+        {
+            // If the token immediately after the macro name is not a '(',
+            // this macro should not be expanded.
+            break;
+        }
+
+        pushMacro(macro, *token);
+    }
+}
+
+void MacroExpander::getToken(Token* token)
+{
+    if (mReserveToken.get())
+    {
+        *token = *mReserveToken;
+        mReserveToken.reset();
+        return;
+    }
+
+    // First pop all empty macro contexts.
+    while (!mContextStack.empty() && mContextStack.back()->empty())
+    {
+        popMacro();
+    }
+
+    if (!mContextStack.empty())
+    {
+        *token = mContextStack.back()->get();
+    }
+    else
+    {
+        mLexer->lex(token);
+    }
+}
+
+void MacroExpander::ungetToken(const Token& token)
+{
+    if (!mContextStack.empty())
+    {
+        MacroContext* context = mContextStack.back();
+        context->unget();
+        assert(context->index >= 0);
+        assert(context->replacements[context->index] == token);
+    }
+    else
+    {
+        assert(!mReserveToken.get());
+        mReserveToken.reset(new Token(token));
+    }
+}
+
+bool MacroExpander::isNextTokenLeftParen()
+{
+    Token token;
+    getToken(&token);
+
+    bool lparen = token.type == '(';
+    ungetToken(token);
+
+    return lparen;
+}
+
+bool MacroExpander::pushMacro(const Macro& macro, const Token& identifier)
+{
+    assert(!macro.disabled);
+    assert(!identifier.expansionDisabled());
+    assert(identifier.type == Token::IDENTIFIER);
+    assert(identifier.value == macro.name);
+
+    std::vector<Token> replacements;
+    if (!expandMacro(macro, identifier, &replacements))
+        return false;
+
+    // Macro is disabled for expansion until it is popped off the stack.
+    macro.disabled = true;
+
+    MacroContext* context = new MacroContext;
+    context->macro = &macro;
+    context->replacements.swap(replacements);
+    mContextStack.push_back(context);
+    return true;
+}
+
+void MacroExpander::popMacro()
+{
+    assert(!mContextStack.empty());
+
+    MacroContext* context = mContextStack.back();
+    mContextStack.pop_back();
+
+    assert(context->empty());
+    assert(context->macro->disabled);
+    context->macro->disabled = false;
+    delete context;
+}
+
+bool MacroExpander::expandMacro(const Macro& macro,
+                                const Token& identifier,
+                                std::vector<Token>* replacements)
+{
+    if (macro.type == Macro::kTypeObj)
+    {
+        replacements->assign(macro.replacements.begin(),
+                             macro.replacements.end());
+    }
+    else
+    {
+        assert(macro.type == Macro::kTypeFunc);
+        std::vector<MacroArg> args;
+        args.reserve(macro.parameters.size());
+        if (!collectMacroArgs(macro, identifier, &args))
+            return false;
+
+        replaceMacroParams(macro, args, replacements);
+    }
+
+    for (size_t i = 0; i < replacements->size(); ++i)
+    {
+        Token& repl = replacements->at(i);
+        if (i == 0)
+        {
+            // The first token in the replacement list inherits the padding
+            // properties of the identifier token.
+            repl.setAtStartOfLine(identifier.atStartOfLine());
+            repl.setHasLeadingSpace(identifier.hasLeadingSpace());
+        }
+        repl.location = identifier.location;
+    }
+    return true;
+}
+
+bool MacroExpander::collectMacroArgs(const Macro& macro,
+                                     const Token& identifier,
+                                     std::vector<MacroArg>* args)
+{
+    Token token;
+    getToken(&token);
+    assert(token.type == '(');
+
+    args->push_back(MacroArg());
+    for (int openParens = 1; openParens != 0; )
+    {
+        getToken(&token);
+
+        if (token.type == Token::LAST)
+        {
+            mDiagnostics->report(Diagnostics::MACRO_UNTERMINATED_INVOCATION,
+                                 identifier.location, identifier.value);
+            // Do not lose EOF token.
+            ungetToken(token);
+            return false;
+        }
+
+        bool isArg = false; // True if token is part of the current argument.
+        switch (token.type)
+        {
+          case '(':
+            ++openParens;
+            isArg = true;
+            break;
+          case ')':
+            --openParens;
+            isArg = openParens != 0;
+            break;
+          case ',':
+            // The individual arguments are separated by comma tokens, but
+            // the comma tokens between matching inner parentheses do not
+            // seperate arguments.
+            if (openParens == 1) args->push_back(MacroArg());
+            isArg = openParens != 1;
+            break;
+          default:
+            isArg = true;
+            break;
+        }
+        if (isArg)
+        {
+            MacroArg& arg = args->back();
+            // Initial whitespace is not part of the argument.
+            if (arg.empty()) token.setHasLeadingSpace(false);
+            arg.push_back(token);
+        }
+    }
+
+    const Macro::Parameters& params = macro.parameters;
+    // If there is only one empty argument, it is equivalent to no argument.
+    if (params.empty() && (args->size() == 1) && args->front().empty())
+    {
+        args->clear();
+    }
+    // Validate the number of arguments.
+    if (args->size() != params.size())
+    {
+        Diagnostics::ID id = args->size() < macro.parameters.size() ?
+            Diagnostics::MACRO_TOO_FEW_ARGS :
+            Diagnostics::MACRO_TOO_MANY_ARGS;
+        mDiagnostics->report(id, identifier.location, identifier.value);
+        return false;
+    }
+
+    // Pre-expand each argument before substitution.
+    // This step expands each argument individually before they are
+    // inserted into the macro body.
+    for (size_t i = 0; i < args->size(); ++i)
+    {
+        MacroArg& arg = args->at(i);
+        TokenLexer lexer(&arg);
+        MacroExpander expander(&lexer, mMacroSet, mDiagnostics);
+
+        arg.clear();
+        expander.lex(&token);
+        while (token.type != Token::LAST)
+        {
+            arg.push_back(token);
+            expander.lex(&token);
+        }
+    }
+    return true;
+}
+
+void MacroExpander::replaceMacroParams(const Macro& macro,
+                                       const std::vector<MacroArg>& args,
+                                       std::vector<Token>* replacements)
+{
+    for (size_t i = 0; i < macro.replacements.size(); ++i)
+    {
+        const Token& repl = macro.replacements[i];
+        if (repl.type != Token::IDENTIFIER)
+        {
+            replacements->push_back(repl);
+            continue;
+        }
+
+        // TODO(alokp): Optimize this.
+        // There is no need to search for macro params every time.
+        // The param index can be cached with the replacement token.
+        Macro::Parameters::const_iterator iter = std::find(
+            macro.parameters.begin(), macro.parameters.end(), repl.value);
+        if (iter == macro.parameters.end())
+        {
+            replacements->push_back(repl);
+            continue;
+        }
+
+        size_t iArg = std::distance(macro.parameters.begin(), iter);
+        const MacroArg& arg = args[iArg];
+        if (arg.empty())
+        {
+            continue;
+        }
+        size_t iRepl = replacements->size();
+        replacements->insert(replacements->end(), arg.begin(), arg.end());
+        // The replacement token inherits padding properties from
+        // macro replacement token.
+        replacements->at(iRepl).setHasLeadingSpace(repl.hasLeadingSpace());
+    }
 }
 
 }  // namespace pp

src/compiler/preprocessor/new/MacroExpander.h

@@ -7,6 +7,8 @@
 #ifndef COMPILER_PREPROCESSOR_MACRO_EXPANDER_H_
 #define COMPILER_PREPROCESSOR_MACRO_EXPANDER_H_
 
+#include <vector>
+
 #include "Lexer.h"
 #include "Macro.h"
 #include "pp_utils.h"
@@ -20,15 +22,50 @@ class MacroExpander : public Lexer
 {
   public:
     MacroExpander(Lexer* lexer, MacroSet* macroSet, Diagnostics* diagnostics);
+    virtual ~MacroExpander();
 
     virtual void lex(Token* token);
 
   private:
     PP_DISALLOW_COPY_AND_ASSIGN(MacroExpander);
 
+    void getToken(Token* token);
+    void ungetToken(const Token& token);
+    bool isNextTokenLeftParen();
+
+    bool pushMacro(const Macro& macro, const Token& identifier);
+    void popMacro();
+
+    bool expandMacro(const Macro& macro,
+                     const Token& identifier,
+                     std::vector<Token>* replacements);
+
+    typedef std::vector<Token> MacroArg;
+    bool collectMacroArgs(const Macro& macro,
+                          const Token& identifier,
+                          std::vector<MacroArg>* args);
+    void replaceMacroParams(const Macro& macro,
+                            const std::vector<MacroArg>& args,
+                            std::vector<Token>* replacements);
+
+    struct MacroContext
+    {
+        const Macro* macro;
+        size_t index;
+        std::vector<Token> replacements;
+
+        MacroContext() : macro(0), index(0) { }
+        bool empty() const { return index == replacements.size(); }
+        const Token& get() { return replacements[index++]; }
+        void unget() { --index; }
+    };
+
     Lexer* mLexer;
     MacroSet* mMacroSet;
     Diagnostics* mDiagnostics;
+
+    std::auto_ptr<Token> mReserveToken;
+    std::vector<MacroContext*> mContextStack;
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/Token.cpp

@@ -41,6 +41,14 @@ void Token::setHasLeadingSpace(bool space)
         flags &= ~HAS_LEADING_SPACE;
 }
 
+void Token::setExpansionDisabled(bool disable)
+{
+    if (disable)
+        flags |= EXPANSION_DISABLED;
+    else
+        flags &= ~EXPANSION_DISABLED;
+}
+
 std::ostream& operator<<(std::ostream& out, const Token& token)
 {
     if (token.hasLeadingSpace())

src/compiler/preprocessor/new/Token.h

@@ -50,8 +50,9 @@ struct Token
     };
     enum Flags
     {
-        AT_START_OF_LINE  = 1 << 0,
-        HAS_LEADING_SPACE = 1 << 1
+        AT_START_OF_LINE   = 1 << 0,
+        HAS_LEADING_SPACE  = 1 << 1,
+        EXPANSION_DISABLED = 1 << 2
     };
 
     Token() : type(0), flags(0) { }
@@ -67,8 +68,11 @@ struct Token
     bool hasLeadingSpace() const { return (flags & HAS_LEADING_SPACE) != 0; }
     void setHasLeadingSpace(bool space);
 
+    bool expansionDisabled() const { return (flags & EXPANSION_DISABLED) != 0; }
+    void setExpansionDisabled(bool disable);
+
     int type;
-    int flags;
+    unsigned int flags;
     SourceLocation location;
     std::string value;
 };

src/compiler/preprocessor/new/Tokenizer.cpp

@@ -2305,6 +2305,7 @@ bool Tokenizer::init(int count, const char* const string[], const int length[])
 void Tokenizer::lex(Token* token)
 {
     token->type = pplex(&token->value,&token->location,mHandle);
+    token->flags = 0;
 
     token->setAtStartOfLine(mContext.lineStart);
     mContext.lineStart = token->type == '\n';

src/compiler/preprocessor/new/Tokenizer.l

@@ -289,6 +289,7 @@ bool Tokenizer::init(int count, const char* const string[], const int length[])
 void Tokenizer::lex(Token* token)
 {
     token->type = yylex(&token->value, &token->location, mHandle);
+    token->flags = 0;
 
     token->setAtStartOfLine(mContext.lineStart);
     mContext.lineStart = token->type == '\n';

tests/build_tests.gyp

@@ -44,6 +44,7 @@
         '../third_party/googlemock/src/gmock_main.cc',
         'preprocessor_tests/char_test.cpp',
         'preprocessor_tests/comment_test.cpp',
+        'preprocessor_tests/define_test.cpp',
         'preprocessor_tests/error_test.cpp',
         'preprocessor_tests/extension_test.cpp',
         'preprocessor_tests/identifier_test.cpp',

tests/preprocessor_tests/define_test.cpp

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "PreprocessorTest.h"
+#include "Token.h"
+
+class DefineTest : public PreprocessorTest
+{
+};
+
+TEST_F(DefineTest, NonIdentifier)
+{
+    const char* input = "#define 2 foo\n"
+                        "2\n";
+    const char* expected = "\n"
+                           "2\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::UNEXPECTED_TOKEN,
+                      pp::SourceLocation(0, 1),
+                      "2"));
+
+    preprocess(input, expected);
+};
+
+TEST_F(DefineTest, ReservedUnderScore1)
+{
+    const char* input = "#define __foo bar\n"
+                        "__foo\n";
+    const char* expected = "\n"
+                           "__foo\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_NAME_RESERVED,
+                      pp::SourceLocation(0, 1),
+                      "__foo"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ReservedUnderScore2)
+{
+    const char* input = "#define foo__bar baz\n"
+                        "foo__bar\n";
+    const char* expected = "\n"
+                           "foo__bar\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_NAME_RESERVED,
+                      pp::SourceLocation(0, 1),
+                      "foo__bar"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ReservedGL)
+{
+    const char* input = "#define GL_foo bar\n"
+                        "GL_foo\n";
+    const char* expected = "\n"
+                           "GL_foo\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_NAME_RESERVED,
+                      pp::SourceLocation(0, 1),
+                      "GL_foo"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjRedefineValid)
+{
+    const char* input = "#define foo (1-1)\n"
+                        "#define foo /* whitespace */ (1-1) /* other */ \n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "\n"
+                           "(1-1)\n";
+    // No error or warning.
+    using testing::_;
+    EXPECT_CALL(mDiagnostics, print(_, _, _)).Times(0);
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjRedefineInvalid)
+{
+    const char* input = "#define foo (0)\n"
+                        "#define foo (1-1)\n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "\n"
+                           "(0)\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_REDEFINED,
+                      pp::SourceLocation(0, 2),
+                      "foo"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncRedefineValid)
+{
+    const char* input = "#define foo(a) ( a )\n"
+                        "#define foo( a )( /* whitespace */ a /* other */ )\n"
+                        "foo(b)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "( b )\n";
+    // No error or warning.
+    using testing::_;
+    EXPECT_CALL(mDiagnostics, print(_, _, _)).Times(0);
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncRedefineInvalid)
+{
+    const char* input = "#define foo(b) ( a )\n"
+                        "#define foo(b) ( b )\n"
+                        "foo(1)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "( a )\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_REDEFINED,
+                      pp::SourceLocation(0, 2),
+                      "foo"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjBasic)
+{
+    const char* input = "#define foo 1\n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjEmpty)
+{
+    const char* input = "#define foo\n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjChain)
+{
+    const char* input = "#define foo 1\n"
+                        "#define bar foo\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjChainReverse)
+{
+    const char* input = "#define bar foo\n"
+                        "#define foo 1\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjRecursive)
+{
+    const char* input = "#define foo bar\n"
+                        "#define bar baz\n"
+                        "#define baz foo\n"
+                        "foo\n"
+                        "bar\n"
+                        "baz\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "foo\n"
+                           "bar\n"
+                           "baz\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjCompositeChain)
+{
+    const char* input = "#define foo 1\n"
+                        "#define bar a foo\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "a 1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjCompositeChainReverse)
+{
+    const char* input = "#define bar a foo\n"
+                        "#define foo 1\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "a 1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjCompositeRecursive)
+{
+    const char* input = "#define foo a bar\n"
+                        "#define bar b baz\n"
+                        "#define baz c foo\n"
+                        "foo\n"
+                        "bar\n"
+                        "baz\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "a b c foo\n"
+                           "b c a bar\n"
+                           "c a b baz\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjChainSelfRecursive)
+{
+    const char* input = "#define foo foo\n"
+                        "#define bar foo\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "foo\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjectLikeWithParens)
+{
+    const char* input = "#define foo ()1\n"
+                        "foo()\n"
+                        "#define bar ()2\n"
+                        "bar()\n";
+    const char* expected = "\n"
+                           "()1()\n"
+                           "\n"
+                           "()2()\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncEmpty)
+{
+    const char* input = "#define foo()\n"
+                        "foo()\n";
+    const char* expected = "\n"
+                           "\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncNoArgs)
+{
+    const char* input = "#define foo() bar\n"
+                        "foo()\n";
+    const char* expected = "\n"
+                           "bar\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncOneArgUnused)
+{
+    const char* input = "#define foo(x) 1\n"
+                        "foo(bar)\n";
+    const char* expected = "\n"
+                           "1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncTwoArgsUnused)
+{
+    const char* input = "#define foo(x,y) 1\n"
+                        "foo(bar,baz)\n";
+    const char* expected = "\n"
+                           "1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncOneArg)
+{
+    const char* input = "#define foo(x) ((x)+1)\n"
+                        "foo(bar)\n";
+    const char* expected = "\n"
+                           "((bar)+1)\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncTwoArgs)
+{
+    const char* input = "#define foo(x,y) ((x)*(y))\n"
+                        "foo(bar,baz)\n";
+    const char* expected = "\n"
+                           "((bar)*(baz))\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncEmptyArgs)
+{
+    const char* input = "#define zero() pass\n"
+                        "#define one(x) pass\n"
+                        "#define two(x,y) pass\n"
+                        "zero()\n"
+                        "one()\n"
+                        "two(,)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "pass\n"
+                           "pass\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncMacroAsParam)
+{
+    const char* input = "#define x 0\n"
+                        "#define foo(x) x\n"
+                        "foo(1)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncOneArgMulti)
+{
+    const char* input = "#define foo(x) (x)\n"
+                        "foo(this is a multi-word argument)\n";
+    const char* expected = "\n"
+                           "(this is a multi-word argument)\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncTwoArgsMulti)
+{
+    const char* input = "#define foo(x,y) x,two fish,red fish,y\n"
+                        "foo(one fish, blue fish)\n";
+    const char* expected = "\n"
+                           "one fish,two fish,red fish,blue fish\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncCompose)
+{
+    const char* input = "#define bar(x) (1+(x))\n"
+                        "#define foo(y) (2*(y))\n"
+                        "foo(bar(3))\n";
+    const char* expected = "\n"
+                           "\n"
+                           "(2*((1+(3))))\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncArgWithParens)
+{
+    const char* input = "#define foo(x) (x)\n"
+                        "foo(argument(with parens) FTW)\n";
+    const char* expected = "\n"
+                           "(argument(with parens) FTW)\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncMacroAsNonMacro)
+{
+    const char* input = "#define foo(bar) bar\n"
+                        "foo bar\n";
+    const char* expected = "\n"
+                           "foo bar\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncExtraNewlines)
+{
+    const char* input = "#define foo(a) (a)\n"
+                        "foo\n"
+                        "(\n"
+                        "1\n"
+                        ")\n";
+    const char* expected = "\n"
+                           "(1)\n"
+                           "\n"
+                           "\n"
+                           "\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToFunc)
+{
+    const char* input = "#define foo() pass\n"
+                        "#define bar foo()\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToNonFunc)
+{
+    const char* input = "#define pass() fail\n"
+                        "#define bar pass\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToFuncWithArgs)
+{
+    const char* input = "#define foo(fail) fail\n"
+                        "#define bar foo(pass)\n"
+                        "bar\n";
+    const char* expected = "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToFuncCompose)
+{
+    const char* input = "#define baz(fail) fail\n"
+                        "#define bar(fail) fail\n"
+                        "#define foo bar(baz(pass))\n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToFuncParensInText1)
+{
+    const char* input = "#define fail() pass\n"
+                        "#define foo fail\n"
+                        "foo()\n";
+    const char* expected = "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToFuncParensInText2)
+{
+    const char* input = "#define bar with,embedded,commas\n"
+                        "#define func(x) pass\n"
+                        "#define foo func\n"
+                        "foo(bar)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainObjToFuncMultiLevel)
+{
+    const char* input = "#define foo(x) pass\n"
+                        "#define bar foo\n"
+                        "#define baz bar\n"
+                        "#define joe baz\n"
+                        "joe (fail)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ObjToFuncRecursive)
+{
+    const char* input = "#define A(a,b) B(a,b)\n"
+                        "#define C A(0,C)\n"
+                        "C\n";
+    const char* expected = "\n"
+                           "\n"
+                           "B(0,C)\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, ChainFuncToFuncCompose)
+{
+    const char* input = "#define baz(fail) fail\n"
+                        "#define bar(fail) fail\n"
+                        "#define foo() bar(baz(pass))\n"
+                        "foo()\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncSelfRecursive)
+{
+    const char* input = "#define foo(a) foo(2*(a))\n"
+                        "foo(3)\n";
+    const char* expected = "\n"
+                           "foo(2*(3))\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncSelfCompose)
+{
+    const char* input = "#define foo(a) foo(2*(a))\n"
+                        "foo(foo(3))\n";
+    const char* expected = "\n"
+                           "foo(2*(foo(2*(3))))\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncSelfComposeNonFunc)
+{
+    const char* input = "#define foo(bar) bar\n"
+                        "foo(foo)\n";
+    const char* expected = "\n"
+                           "foo\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncSelfComposeNonFuncMultiTokenArg)
+{
+    const char* input = "#define foo(bar) bar\n"
+                        "foo(1+foo)\n";
+    const char* expected = "\n"
+                           "1+foo\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FinalizeUnexpandedMacro)
+{
+    const char* input = "#define expand(x) expand(x once)\n"
+                        "#define foo(x) x\n"
+                        "foo(expand(just))\n";
+    const char* expected = "\n"
+                           "\n"
+                           "expand(just once)\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncArgWithCommas)
+{
+    const char* input = "#define foo(x) pass\n"
+                        "foo(argument (with,embedded, commas) -- baz)\n";
+    const char* expected = "\n"
+                           "pass\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncArgObjMaroWithComma)
+{
+    const char* input = "#define foo(a) (a)\n"
+                        "#define bar two,words\n"
+                        "foo(bar)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "(two,words)\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncLeftParenInMacroRightParenInText)
+{
+    const char* input = "#define bar(a) a*2\n"
+                        "#define foo bar(\n"
+                        "foo b)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "b*2\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, RepeatedArg)
+{
+    const char* input = "#define double(x) x x\n"
+                        "double(1)\n";
+    const char* expected = "\n"
+                           "1 1\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncMissingRightParen)
+{
+    const char* input = "#define foo(x) (2*(x))\n"
+                        "foo(3\n";
+    const char* expected = "\n"
+                           "\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_UNTERMINATED_INVOCATION,
+                      pp::SourceLocation(0, 2),
+                      "foo"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, FuncIncorrectArgCount)
+{
+    const char* input = "#define foo(x,y) ((x)+(y))\n"
+                        "foo()\n"
+                        "foo(1)\n"
+                        "foo(1,2,3)\n";
+    const char* expected = "\n"
+                           "\n"
+                           "\n"
+                           "\n";
+
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_TOO_FEW_ARGS,
+                      pp::SourceLocation(0, 2),
+                      "foo"));
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_TOO_FEW_ARGS,
+                      pp::SourceLocation(0, 3),
+                      "foo"));
+    EXPECT_CALL(mDiagnostics,
+                print(pp::Diagnostics::MACRO_TOO_MANY_ARGS,
+                      pp::SourceLocation(0, 4),
+                      "foo"));
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, Undef)
+{
+    const char* input = "#define foo 1\n"
+                        "foo\n"
+                        "#undef foo\n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "1\n"
+                           "\n"
+                           "foo\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, UndefRedefine)
+{
+    const char* input = "#define foo 1\n"
+                        "foo\n"
+                        "#undef foo\n"
+                        "foo\n"
+                        "#define foo 2\n"
+                        "foo\n";
+    const char* expected = "\n"
+                           "1\n"
+                           "\n"
+                           "foo\n"
+                           "\n"
+                           "2\n";
+
+    preprocess(input, expected);
+}
+
+TEST_F(DefineTest, C99Example)
+{
+    const char* input =
+        "#define x    3          \n"
+        "#define f(a) f(x * (a)) \n"
+        "#undef  x               \n"
+        "#define x    2          \n"
+        "#define g    f          \n"
+        "#define z    z[0]       \n"
+        "#define h    g(~        \n"
+        "#define m(a) a(w)       \n"
+        "#define w    0,1        \n"
+        "#define t(a) a          \n"
+        "#define p()  int        \n"
+        "#define q(x) x          \n"
+        "                        \n"
+        "f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);\n"
+        "g(x+(3,4)-w) | h 5) & m\n"
+        "    (f)^m(m);\n"
+        "p() i[q()] = { q(1), 23, 4, 5, };\n";
+    const char* expected = 
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1);\n"
+        "f(2 * (2+(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))\n"
+        "^m(0,1);\n"
+        "int i[] = { 1, 23, 4, 5, };\n";
+
+    preprocess(input, expected);
+}
+