Complete implementation for handling #define directive.

src/compiler/preprocessor/new/Context.cpp

@@ -13,6 +13,21 @@
 #include "stl_utils.h"
 #include "token_type.h"
 
+static bool isMacroNameReserved(const std::string* name)
+{
+    ASSERT(name);
+
+    // Names prefixed with "GL_" are reserved.
+    if (name->substr(0, 3) == "GL_")
+        return true;
+
+    // Names containing two consecutive underscores are reserved.
+    if (name->find("__") != std::string::npos)
+        return true;
+
+    return false;
+}
+
 namespace pp
 {
 
@@ -58,18 +73,29 @@ bool Context::process(int count,
 
 bool Context::defineMacro(pp::Token::Location location,
                           pp::Macro::Type type,
-                          std::string* identifier,
-                          pp::Macro::ParameterVector* parameters,
+                          std::string* name,
+                          pp::TokenVector* parameters,
                           pp::TokenVector* replacements)
 {
-    // TODO(alokp): Check for reserved macro names and duplicate macros.
-    mMacros[*identifier] = new Macro(type, identifier, parameters, replacements);
+    std::auto_ptr<Macro> macro(new Macro(type, name, parameters, replacements));
+    if (isMacroNameReserved(name))
+    {
+        // TODO(alokp): Report error.
+        return false;
+    }
+    if (isMacroDefined(name))
+    {
+        // TODO(alokp): Report error.
+        return false;
+    }
+
+    mMacros[*name] = macro.release();
     return true;
 }
 
-bool Context::undefineMacro(const std::string* identifier)
+bool Context::undefineMacro(const std::string* name)
 {
-    MacroSet::iterator iter = mMacros.find(*identifier);
+    MacroSet::iterator iter = mMacros.find(*name);
     if (iter == mMacros.end())
     {
         // TODO(alokp): Report error.
@@ -79,9 +105,9 @@ bool Context::undefineMacro(const std::string* identifier)
     return true;
 }
 
-bool Context::isMacroDefined(const std::string* identifier)
+bool Context::isMacroDefined(const std::string* name)
 {
-    return mMacros.find(*identifier) != mMacros.end();
+    return mMacros.find(*name) != mMacros.end();
 }
 
 // Reset to initialized state.
@@ -96,15 +122,15 @@ void Context::reset()
     mOutput = NULL;
 }
 
-void Context::defineBuiltInMacro(const std::string& identifier, int value)
+void Context::defineBuiltInMacro(const std::string& name, int value)
 {
     std::ostringstream stream;
     stream << value;
     Token* token = new Token(0, INT_CONSTANT, new std::string(stream.str()));
     TokenVector* replacements = new pp::TokenVector(1, token);
 
-    mMacros[identifier] = new Macro(Macro::kTypeObj,
-                                    new std::string(identifier),
+    mMacros[name] = new Macro(Macro::kTypeObj,
+                              new std::string(name),
                               NULL,
                               replacements);
 }

src/compiler/preprocessor/new/Context.h

@@ -33,11 +33,11 @@ class Context
 
     bool defineMacro(pp::Token::Location location,
                      pp::Macro::Type type,
-                     std::string* identifier,
-                     pp::Macro::ParameterVector* parameters,
+                     std::string* name,
+                     pp::TokenVector* parameters,
                      pp::TokenVector* replacements);
-    bool undefineMacro(const std::string* identifier);
-    bool isMacroDefined(const std::string* identifier);
+    bool undefineMacro(const std::string* name);
+    bool isMacroDefined(const std::string* name);
 
   private:
     DISALLOW_COPY_AND_ASSIGN(Context);
@@ -46,7 +46,7 @@ class Context
     void reset();
     bool initLexer();
     void destroyLexer();
-    void defineBuiltInMacro(const std::string& identifier, int value);
+    void defineBuiltInMacro(const std::string& name, int value);
     bool parse();
 
     void* mLexer;  // Lexer handle.

src/compiler/preprocessor/new/Macro.cpp

@@ -14,11 +14,11 @@ namespace pp
 {
 
 Macro::Macro(Type type,
-             std::string* identifier,
-             ParameterVector* parameters,
+             std::string* name,
+             TokenVector* parameters,
              TokenVector* replacements)
     : mType(type),
-      mIdentifier(identifier),
+      mName(name),
       mParameters(parameters),
       mReplacements(replacements)
 {
@@ -26,7 +26,7 @@ Macro::Macro(Type type,
 
 Macro::~Macro()
 {
-    delete mIdentifier;
+    delete mName;
 
     if (mParameters)
     {

src/compiler/preprocessor/new/Macro.h

@@ -24,26 +24,25 @@ class Macro
         kTypeObj,
         kTypeFunc
     };
-    typedef std::vector<std::string*> ParameterVector;
 
     // Takes ownership of pointer parameters.
     Macro(Type type,
-          std::string* identifier,
-          ParameterVector* parameters,
+          std::string* name,
+          TokenVector* parameters,
           TokenVector* replacements);
     ~Macro();
 
     Type type() const { return mType; }
-    const std::string* identifier() const { return mIdentifier; }
-    const ParameterVector* parameters() const { return mParameters; }
+    const std::string* identifier() const { return mName; }
+    const TokenVector* parameters() const { return mParameters; }
     const TokenVector* replacements() const { return mReplacements; }
 
   private:
     DISALLOW_COPY_AND_ASSIGN(Macro);
 
     Type mType;
-    std::string* mIdentifier;
-    ParameterVector* mParameters;
+    std::string* mName;
+    TokenVector* mParameters;
     TokenVector* mReplacements;
 };
 

src/compiler/preprocessor/new/Token.cpp

@@ -40,9 +40,6 @@ std::ostream& operator<<(std::ostream& out, const Token& token)
 {
     switch (token.type())
     {
-      case SPACE:
-        out << " ";
-        break;
       case INT_CONSTANT:
       case FLOAT_CONSTANT:
       case IDENTIFIER:

src/compiler/preprocessor/new/pp.l

@@ -36,6 +36,8 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 
 #define YY_INPUT(buf, result, maxSize) \
     result = yyextra->readInput(buf, maxSize);
+    
+static std::string* extractMacroName(const char* str, int len);
 %}
 
 %option noyywrap nounput never-interactive
@@ -61,9 +63,15 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 {HASH} { return HASH; }
 
-{HASH}define { return HASH_DEFINE_OBJ; }
-{HASH}define{HSPACE}+/{IDENTIFIER}"(" { return HASH_DEFINE_FUNC; }
-{HASH}undef { return HASH_UNDEF; }
+{HASH}define{HSPACE}+{IDENTIFIER}/[ \t\n] {
+    yylval->sval = extractMacroName(yytext, yyleng);
+    return HASH_DEFINE_OBJ;
+}
+{HASH}define{HSPACE}+{IDENTIFIER}/"(" {
+    yylval->sval = extractMacroName(yytext, yyleng);
+    return HASH_DEFINE_FUNC;
+}
+{HASH}undef{HSPACE}+ { return HASH_UNDEF; }
 
 {HASH}if { return HASH_IF; }
 {HASH}ifdef { return HASH_IFDEF; }
@@ -96,7 +104,7 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 {PUNCTUATOR} { return yytext[0]; }
 
-{HSPACE}+ { return SPACE; }
+[ \t\v\f]+ { /* Ignore whitespace */ }
 
 \n {
     ++yylineno; yycolumn = 0;
@@ -107,6 +115,25 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 %%
 
+std::string* extractMacroName(const char* str, int len)
+{
+    // The input string is of the form {HASH}define{HSPACE}+{IDENTIFIER}
+    // We just need to find the last HSPACE.
+    ASSERT(str && (len > 8));  // strlen("#define ") == 8;
+
+    std::string* name = NULL;
+    for (int i = len - 1; i >= 0; --i)
+    {
+        if ((str[i] == ' ') || (str[i] == '\t'))
+        {
+            name = new std::string(str + i + 1, len - i - 1);
+            break;
+        }
+    }
+    ASSERT(name);
+    return name;
+}
+
 namespace pp {
 
 int Context::readInput(char* buf, int maxSize)

src/compiler/preprocessor/new/pp.y

@@ -36,7 +36,6 @@ WHICH GENERATES THE GLSL ES PARSER.
 %union {
     int ival;
     std::string* sval;
-    std::vector<std::string*>* slist;
     pp::Token* tval;
     pp::TokenVector* tlist;
 }
@@ -51,15 +50,14 @@ static void pushConditionalBlock(pp::Context* context, bool condition);
 static void popConditionalBlock(pp::Context* context);
 %}
 
-%token HASH HASH_DEFINE_OBJ HASH_DEFINE_FUNC HASH_UNDEF
+%token HASH HASH_UNDEF
 %token HASH_IF HASH_IFDEF HASH_IFNDEF HASH_ELSE HASH_ELIF HASH_ENDIF DEFINED
 %token HASH_ERROR HASH_PRAGMA HASH_EXTENSION HASH_VERSION HASH_LINE
-%token SPACE
+%token <sval> HASH_DEFINE_OBJ HASH_DEFINE_FUNC
 %token <sval> INT_CONSTANT FLOAT_CONSTANT IDENTIFIER
 %type <ival> operator
-%type <slist> parameter_list
 %type <tval> conditional_token token
-%type <tlist> text_line replacement_token_list conditional_token_list token_list
+%type <tlist> parameter_list replacement_list conditional_list token_list
 %%
 
 input
@@ -68,32 +66,32 @@ input
 ;
 
 line
-    : text_line {
+    : text_line
+    | control_line
+;
+
+text_line
+    : '\n'
+    | token_list '\n' {
         // TODO(alokp): Expand macros.
         pp::TokenVector* out = context->output();
         out->insert(out->end(), $1->begin(), $1->end());
         delete $1;
     }
-    | control_line
-;
-
-text_line
-    : '\n' { $$ = NULL; }
-    | token_list '\n' { $$ = $1; }
 ;
 
 control_line
     : HASH '\n'
-    | HASH_DEFINE_OBJ IDENTIFIER replacement_token_list '\n' {
-        context->defineMacro(@2.first_line, pp::Macro::kTypeObj, $2, NULL, $3);
+    | HASH_DEFINE_OBJ replacement_list '\n' {
+        context->defineMacro(@1.first_line, pp::Macro::kTypeObj, $1, NULL, $2);
     }
-    | HASH_DEFINE_FUNC IDENTIFIER '(' parameter_list ')' replacement_token_list '\n' {
-        context->defineMacro(@2.first_line, pp::Macro::kTypeFunc, $2, $4, $6);
+    | HASH_DEFINE_FUNC '(' parameter_list ')' replacement_list '\n' {
+        context->defineMacro(@1.first_line, pp::Macro::kTypeFunc, $1, $3, $5);
     }
     | HASH_UNDEF IDENTIFIER '\n' {
         context->undefineMacro($2);
     }
-    | HASH_IF conditional_token_list '\n' {
+    | HASH_IF conditional_list '\n' {
         pushConditionalBlock(context, $2 ? true : false);
     }
     | HASH_IFDEF IDENTIFIER '\n' {
@@ -102,7 +100,7 @@ control_line
     | HASH_IFNDEF IDENTIFIER '\n' {
         pushConditionalBlock(context, !context->isMacroDefined($2));
     }
-    | HASH_ELIF conditional_token_list '\n' {
+    | HASH_ELIF conditional_list '\n' {
     }
     | HASH_ELSE '\n' {
     }
@@ -116,39 +114,29 @@ control_line
     | HASH_LINE '\n'
 ;
 
-replacement_token_list
-    : /* empty */ { $$ = NULL }
+replacement_list
+    : /* empty */ { $$ = NULL; }
     | token_list
-;
 
-conditional_token_list
-    : conditional_token {
+parameter_list
+    : /* empty */ { $$ = NULL; }
+    | IDENTIFIER {
         $$ = new pp::TokenVector;
-        $$->push_back($1);
+        $$->push_back(new pp::Token(@1.first_line, IDENTIFIER, $1));
     }
-    | conditional_token_list conditional_token {
+    | parameter_list ',' IDENTIFIER {
         $$ = $1;
-        $$->push_back($2);
-    }
-;
-
-conditional_token
-    : DEFINED IDENTIFIER {
-    }
-    | DEFINED '(' IDENTIFIER ')' {
+        $$->push_back(new pp::Token(@3.first_line, IDENTIFIER, $3));
     }
-    | token
-;
 
-parameter_list
-    : /* empty */ { $$ = NULL; }
-    | IDENTIFIER {
-        $$ = new std::vector<std::string*>();
+conditional_list
+    : conditional_token {
+        $$ = new pp::TokenVector;
         $$->push_back($1);
     }
-    | parameter_list ',' IDENTIFIER {
+    | conditional_list conditional_token {
         $$ = $1;
-        $$->push_back($3);
+        $$->push_back($2);
     }
 ;
 
@@ -163,13 +151,18 @@ token_list
     }
 ;
 
+conditional_token
+    : DEFINED IDENTIFIER {
+    }
+    | DEFINED '(' IDENTIFIER ')' {
+    }
+    | token
+;
+
 token
     : operator {
         $$ = new pp::Token(@1.first_line, $1, NULL);
     }
-    | SPACE {
-        $$ = new pp::Token(@1.first_line, SPACE, NULL);
-    }
     | INT_CONSTANT {
         $$ = new pp::Token(@1.first_line, INT_CONSTANT, $1);
     }

src/compiler/preprocessor/new/pp_tab.h

@@ -40,48 +40,46 @@
       know about them.  */
    enum yytokentype {
      HASH = 258,
-     HASH_DEFINE_OBJ = 259,
-     HASH_DEFINE_FUNC = 260,
-     HASH_UNDEF = 261,
-     HASH_IF = 262,
-     HASH_IFDEF = 263,
-     HASH_IFNDEF = 264,
-     HASH_ELSE = 265,
-     HASH_ELIF = 266,
-     HASH_ENDIF = 267,
-     DEFINED = 268,
-     HASH_ERROR = 269,
-     HASH_PRAGMA = 270,
-     HASH_EXTENSION = 271,
-     HASH_VERSION = 272,
-     HASH_LINE = 273,
-     SPACE = 274,
-     INT_CONSTANT = 275,
-     FLOAT_CONSTANT = 276,
-     IDENTIFIER = 277
+     HASH_UNDEF = 259,
+     HASH_IF = 260,
+     HASH_IFDEF = 261,
+     HASH_IFNDEF = 262,
+     HASH_ELSE = 263,
+     HASH_ELIF = 264,
+     HASH_ENDIF = 265,
+     DEFINED = 266,
+     HASH_ERROR = 267,
+     HASH_PRAGMA = 268,
+     HASH_EXTENSION = 269,
+     HASH_VERSION = 270,
+     HASH_LINE = 271,
+     HASH_DEFINE_OBJ = 272,
+     HASH_DEFINE_FUNC = 273,
+     INT_CONSTANT = 274,
+     FLOAT_CONSTANT = 275,
+     IDENTIFIER = 276
    };
 #endif
 /* Tokens.  */
 #define HASH 258
-#define HASH_DEFINE_OBJ 259
-#define HASH_DEFINE_FUNC 260
-#define HASH_UNDEF 261
-#define HASH_IF 262
-#define HASH_IFDEF 263
-#define HASH_IFNDEF 264
-#define HASH_ELSE 265
-#define HASH_ELIF 266
-#define HASH_ENDIF 267
-#define DEFINED 268
-#define HASH_ERROR 269
-#define HASH_PRAGMA 270
-#define HASH_EXTENSION 271
-#define HASH_VERSION 272
-#define HASH_LINE 273
-#define SPACE 274
-#define INT_CONSTANT 275
-#define FLOAT_CONSTANT 276
-#define IDENTIFIER 277
+#define HASH_UNDEF 259
+#define HASH_IF 260
+#define HASH_IFDEF 261
+#define HASH_IFNDEF 262
+#define HASH_ELSE 263
+#define HASH_ELIF 264
+#define HASH_ENDIF 265
+#define DEFINED 266
+#define HASH_ERROR 267
+#define HASH_PRAGMA 268
+#define HASH_EXTENSION 269
+#define HASH_VERSION 270
+#define HASH_LINE 271
+#define HASH_DEFINE_OBJ 272
+#define HASH_DEFINE_FUNC 273
+#define INT_CONSTANT 274
+#define FLOAT_CONSTANT 275
+#define IDENTIFIER 276
 
 
 
@@ -92,7 +90,6 @@ typedef union YYSTYPE
 {
     int ival;
     std::string* sval;
-    std::vector<std::string*>* slist;
     pp::Token* tval;
     pp::TokenVector* tlist;
 }