Modified Token class to store various types of data. Added debug code to dump…

src/compiler/preprocessor/new/Token.cpp

@@ -6,34 +6,53 @@
 
 #include "Token.h"
 
+#include "token_type.h"
+
 static const int kLocationLineSize = 16;  // in bits.
 static const int kLocationLineMask = (1 << kLocationLineSize) - 1;
 
 namespace pp
 {
 
-Token::Token() : mLocation(-1), mType(-1), mValue(0)
+Token::Location Token::encodeLocation(int line, int file)
 {
+    return (file << kLocationLineSize) | (line & kLocationLineMask);
 }
 
-Token::Token(Location l, int t) : mLocation(l), mType(t)
+void Token::decodeLocation(Location loc, int* line, int* file)
 {
+    if (file) *file = loc >> kLocationLineSize;
+    if (line) *line = loc & kLocationLineMask;
 }
 
-Token::Token(Location l, int t, const std::string& s) : mLocation(l), mType(t), mValue(s)
+Token::Token(Location location, int type, std::string* value)
+    : mLocation(location),
+      mType(type),
+      mValue(value)
 {
 }
 
-Token::Location Token::encodeLocation(int line, int file)
-{
-    return (file << kLocationLineSize) | (line & kLocationLineMask);
+Token::~Token() {
+    delete mValue;
 }
 
-void Token::decodeLocation(Location loc, int* line, int* file)
+std::ostream& operator<<(std::ostream& out, const Token& token)
 {
-    if (file) *file = loc >> kLocationLineSize;
-    if (line) *line = loc & kLocationLineMask;
+    switch (token.type())
+    {
+      case SPACE:
+        out << " ";
+        break;
+      case INT_CONSTANT:
+      case FLOAT_CONSTANT:
+      case IDENTIFIER:
+        out << *(token.value());
+        break;
+      default:
+        out << static_cast<char>(token.type());
+        break;
+    }
+    return out;
 }
-
 }  // namespace pp
 

src/compiler/preprocessor/new/Token.h

@@ -17,27 +17,28 @@ class Token
 {
   public:
     typedef int Location;
-
-    Token();
-    Token(Location l, int t);
-    Token(Location l, int t, const std::string& s);
-
     static Location encodeLocation(int line, int file);
     static void decodeLocation(Location loc, int* line, int* file);
 
+    // Takes ownership of string.
+    Token(Location location, int type, std::string* value);
+    ~Token();
+
     Location location() const { return mLocation; }
     int type() const { return mType; }
-    const std::string& value() const { return mValue; }
+    const std::string* value() const { return mValue; }
 
   private:
     Location mLocation;
     int mType;
-    std::string mValue;
+    std::string* mValue;
 };
 
 typedef std::vector<Token*> TokenVector;
 typedef TokenVector::const_iterator TokenIterator;
 
+extern std::ostream& operator<<(std::ostream& out, const Token& token);
+
 }  // namepsace pp
 #endif  // COMPILER_PREPROCESSOR_TOKEN_H_
 

src/compiler/preprocessor/new/pp.l

@@ -56,6 +56,10 @@ DECIMAL_CONSTANT      [1-9][0-9]*
 OCTAL_CONSTANT        0[0-7]*
 HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+
 
+DIGIT                [0-9]
+EXPONENT_PART        [eE][+-]?{DIGIT}+
+FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
+
 %%
 
 {HASH} { return HASH; }
@@ -79,13 +83,18 @@ HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+
 {HASH}line { return HASH_LINE; }
 
 {IDENTIFIER} {
-    yylval->tval = new pp::Token(yylineno, IDENTIFIER, std::string(yytext, yyleng));
+    yylval->sval = new std::string(yytext, yyleng);
     return IDENTIFIER;
 }
 
 {DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
-    yylval->tval = new pp::Token(yylineno, INTEGER_CONSTANT, std::string(yytext, yyleng));
-    return INTEGER_CONSTANT;
+    yylval->sval = new std::string(yytext, yyleng);
+    return INT_CONSTANT;
+}
+
+({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
+    yylval->sval = new std::string(yytext, yyleng);
+    return FLOAT_CONSTANT;
 }
 
 {PUNCTUATOR} { return yytext[0]; }

src/compiler/preprocessor/new/pp.y

@@ -36,6 +36,8 @@ WHICH GENERATES THE GLSL ES PARSER.
 
 %union {
     int ival;
+    std::string* sval;
+    std::vector<std::string*>* slist;
     pp::Token* tval;
     pp::TokenVector* tlist;
 }
@@ -49,11 +51,11 @@ static void yyerror(YYLTYPE* llocp,
 static void defineMacro(pp::Context* context,
                         YYLTYPE* llocp,
                         pp::Macro::Type type,
-                        pp::Token* identifier,
-                        pp::TokenVector* parameters,
+                        const std::string* identifier,
+                        std::vector<std::string*>* parameters,
                         pp::TokenVector* replacements);
-static void undefineMacro(pp::Context* context, pp::Token* identifier);
-static bool isMacroDefined(pp::Context* context, pp::Token* identifier);
+static void undefineMacro(pp::Context* context, const std::string* identifier);
+static bool isMacroDefined(pp::Context* context, const std::string* identifier);
 static void pushConditionalBlock(pp::Context* context, bool condition);
 static void popConditionalBlock(pp::Context* context);
 %}
@@ -62,10 +64,11 @@ static void popConditionalBlock(pp::Context* context);
 %token HASH_IF HASH_IFDEF HASH_IFNDEF HASH_ELSE HASH_ELIF HASH_ENDIF DEFINED
 %token HASH_ERROR HASH_PRAGMA HASH_EXTENSION HASH_VERSION HASH_LINE
 %token SPACE
-%token <tval> IDENTIFIER INTEGER_CONSTANT FLOAT_CONSTANT
+%token <sval> INT_CONSTANT FLOAT_CONSTANT IDENTIFIER
 %type <ival> operator
+%type <slist> parameter_list
 %type <tval> conditional_token token
-%type <tlist> text_line parameter_list replacement_token_list conditional_token_list token_list
+%type <tlist> text_line replacement_token_list conditional_token_list token_list
 %%
 
 input
@@ -74,7 +77,12 @@ input
 ;
 
 line
-    : text_line
+    : text_line {
+        // TODO(alokp): Expand macros.
+        pp::TokenVector* out = context->output;
+        out->insert(out->end(), $1->begin(), $1->end());
+        delete $1;
+    }
     | control_line
 ;
 
@@ -144,7 +152,7 @@ conditional_token
 parameter_list
     : /* empty */ { $$ = NULL; }
     | IDENTIFIER {
-        $$ = new pp::TokenVector;
+        $$ = new std::vector<std::string*>();
         $$->push_back($1);
     }
     | parameter_list ',' IDENTIFIER {
@@ -155,7 +163,6 @@ parameter_list
 
 token_list
     : token {
-        //context->ppData.skipWS = false;
         $$ = new pp::TokenVector;
         $$->push_back($1);
     }
@@ -166,11 +173,21 @@ token_list
 ;
 
 token
-    : IDENTIFIER
-    | INTEGER_CONSTANT
-    | FLOAT_CONSTANT
-    | SPACE { $$ = new pp::Token(@1.first_line, SPACE); }
-    | operator { $$ = new pp::Token(@1.first_line, $1); }
+    : operator {
+        $$ = new pp::Token(@1.first_line, $1, NULL);
+    }
+    | SPACE {
+        $$ = new pp::Token(@1.first_line, SPACE, NULL);
+    }
+    | INT_CONSTANT {
+        $$ = new pp::Token(@1.first_line, INT_CONSTANT, $1);
+    }
+    | FLOAT_CONSTANT {
+        $$ = new pp::Token(@1.first_line, FLOAT_CONSTANT, $1);
+    }
+    | IDENTIFIER {
+        $$ = new pp::Token(@1.first_line, IDENTIFIER, $1);
+    }
 ;
 
 operator
@@ -209,17 +226,17 @@ void yyerror(YYLTYPE* llocp, pp::Context* context, const char* reason)
 void defineMacro(pp::Context* context,
                  YYLTYPE* llocp,
                  pp::Macro::Type type,
-                 pp::Token* identifier,
-                 pp::TokenVector* parameters,
+                 const std::string* identifier,
+                 std::vector<std::string*>* parameters,
                  pp::TokenVector* replacements)
 {
 }
 
-void undefineMacro(pp::Context* context, pp::Token* identifier)
+void undefineMacro(pp::Context* context, const std::string* identifier)
 {
 }
 
-bool isMacroDefined(pp::Context* context, pp::Token* identifier)
+bool isMacroDefined(pp::Context* context, const std::string* identifier)
 {
     return false;
 }

src/compiler/preprocessor/new/pp_tab.h

@@ -56,9 +56,9 @@
      HASH_VERSION = 272,
      HASH_LINE = 273,
      SPACE = 274,
-     IDENTIFIER = 275,
-     INTEGER_CONSTANT = 276,
-     FLOAT_CONSTANT = 277
+     INT_CONSTANT = 275,
+     FLOAT_CONSTANT = 276,
+     IDENTIFIER = 277
    };
 #endif
 /* Tokens.  */
@@ -79,9 +79,9 @@
 #define HASH_VERSION 272
 #define HASH_LINE 273
 #define SPACE 274
-#define IDENTIFIER 275
-#define INTEGER_CONSTANT 276
-#define FLOAT_CONSTANT 277
+#define INT_CONSTANT 275
+#define FLOAT_CONSTANT 276
+#define IDENTIFIER 277
 
 
 
@@ -91,6 +91,8 @@ typedef union YYSTYPE
 
 {
     int ival;
+    std::string* sval;
+    std::vector<std::string*>* slist;
     pp::Token* tval;
     pp::TokenVector* tlist;
 }

src/compiler/preprocessor/new/token_type.h

+//
+// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_TOKEN_TYPE_H_
+#define COMPILER_PREPROCESSOR_TOKEN_TYPE_H_
+
+#include "pp_tab.h"
+
+#endif  // COMPILER_PREPROCESSOR_TOKEN_TYPE_H_
+