Punted on bison parser. It was turning out to be more complicated than…

src/compiler/preprocessor/new/Lexer.h

@@ -10,8 +10,7 @@
 #include <memory>
 #include "common/angleutils.h"
 
-struct YYLTYPE;
-union YYSTYPE;
+#include "Token.h"
 
 namespace pp
 {
@@ -26,7 +25,7 @@ class Lexer
 
     bool init(int count, const char* const string[], const int length[]);
 
-    int lex(YYSTYPE* lvalp, YYLTYPE* llocp);
+    int lex(Token* token);
 
   private:
     DISALLOW_COPY_AND_ASSIGN(Lexer);
@@ -38,5 +37,5 @@ class Lexer
 };
 
 }  // namespace pp
-#endif  // COMPILER_PREPROCESSOR_BUFFER_LEXER_H_
+#endif  // COMPILER_PREPROCESSOR_LEXER_H_
 

src/compiler/preprocessor/new/Token.cpp

@@ -6,50 +6,13 @@
 
 #include "Token.h"
 
-#include "token_type.h"
-
-static const int kLocationLineSize = 16;  // in bits.
-static const int kLocationLineMask = (1 << kLocationLineSize) - 1;
-
 namespace pp
 {
 
-Token::Location Token::encodeLocation(int line, int file)
-{
-    return (file << kLocationLineSize) | (line & kLocationLineMask);
-}
-
-void Token::decodeLocation(Location loc, int* line, int* file)
-{
-    if (file) *file = loc >> kLocationLineSize;
-    if (line) *line = loc & kLocationLineMask;
-}
-
-Token::Token(Location location, int type, std::string* value)
-    : mLocation(location),
-      mType(type),
-      mValue(value)
-{
-}
-
-Token::~Token() {
-    delete mValue;
-}
-
 std::ostream& operator<<(std::ostream& out, const Token& token)
 {
-    switch (token.type())
-    {
-      case INT_CONSTANT:
-      case FLOAT_CONSTANT:
-      case IDENTIFIER:
-        out << *(token.value());
-        break;
-      default:
-        out << static_cast<char>(token.type());
-        break;
-    }
+    out << token.value;
     return out;
 }
-}  // namespace pp
 
+}  // namespace pp

src/compiler/preprocessor/new/Token.h

@@ -8,41 +8,40 @@
 #define COMPILER_PREPROCESSOR_TOKEN_H_
 
 #include <string>
-#include <vector>
-
-#include "common/angleutils.h"
 
 namespace pp
 {
 
-class Token
+struct Token
 {
-  public:
-    typedef int Location;
-    static Location encodeLocation(int line, int file);
-    static void decodeLocation(Location loc, int* line, int* file);
-
-    // Takes ownership of string.
-    Token(Location location, int type, std::string* value);
-    ~Token();
-
-    Location location() const { return mLocation; }
-    int type() const { return mType; }
-    const std::string* value() const { return mValue; }
-
-  private:
-    DISALLOW_COPY_AND_ASSIGN(Token);
-
-    Location mLocation;
-    int mType;
-    std::string* mValue;
+    enum Type
+    {
+        IDENTIFIER = 258,
+
+        CONST_INT,
+        CONST_FLOAT,
+
+        OP_LEFT_SHIFT,
+        OP_RIGHT_SHIFT,
+        OP_LESS_EQUAL,
+        OP_GREATER_EQUAL,
+        OP_EQUAL,
+        OP_NOT_EQUAL,
+        OP_AND_AND,
+        OP_OR_OR
+    };
+    struct Location
+    {
+        int line;
+        int string;
+    };
+
+    int type;
+    Location location;
+    std::string value;
 };
 
-typedef std::vector<Token*> TokenVector;
-typedef TokenVector::const_iterator TokenIterator;
-
 extern std::ostream& operator<<(std::ostream& out, const Token& token);
 
 }  // namepsace pp
 #endif  // COMPILER_PREPROCESSOR_TOKEN_H_
-

src/compiler/preprocessor/new/pp.l

@@ -23,24 +23,21 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 }
 
 %{
-#include "compiler/debug.h"
 #include "Input.h"
 #include "Lexer.h"
-#include "Token.h"
-#include "token_type.h"
-
-#define YY_USER_ACTION                        \
-    do {                                      \
-        yylloc->first_line = yylineno;        \
-        yylloc->first_column = yycolumn + 1;  \
-        yycolumn += yyleng;                   \
+
+typedef std::string YYSTYPE;
+typedef pp::Token::Location YYLTYPE;
+
+#define YY_USER_ACTION           \
+    do {                         \
+        yylloc->line = yylineno; \
     } while(0);
 
 #define YY_INPUT(buf, result, maxSize) \
     result = readInput(buf, maxSize, yyscanner);
 
 static int readInput(char* buf, int maxSize, yyscan_t scanner);
-static std::string* extractMacroName(const char* str, int len);
 %}
 
 %option noyywrap nounput never-interactive
@@ -48,8 +45,6 @@ static std::string* extractMacroName(const char* str, int len);
 %option prefix="pp"
 %option extra-type="pp::Input*"
 
-HSPACE      [ \t]
-HASH        ^{HSPACE}*#{HSPACE}*
 IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
 PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]
 
@@ -63,45 +58,23 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 %%
 
-{HASH} { return HASH; }
-
-{HASH}define{HSPACE}+{IDENTIFIER}/[ \t\n] {
-    yylval->sval = extractMacroName(yytext, yyleng);
-    return HASH_DEFINE_OBJ;
-}
-{HASH}define{HSPACE}+{IDENTIFIER}/"(" {
-    yylval->sval = extractMacroName(yytext, yyleng);
-    return HASH_DEFINE_FUNC;
+# {
+    return yytext[0];
 }
-{HASH}undef{HSPACE}+ { return HASH_UNDEF; }
-
-{HASH}if { return HASH_IF; }
-{HASH}ifdef { return HASH_IFDEF; }
-{HASH}ifndef { return HASH_IFNDEF; }
-{HASH}else { return HASH_ELSE; }
-{HASH}elif { return HASH_ELIF; }
-{HASH}endif { return HASH_ENDIF; }
-"defined" { return DEFINED; }
-
-{HASH}error { return HASH_ERROR; }
-{HASH}pragma { return HASH_PRAGMA; }
-{HASH}extension { return HASH_EXTENSION; }
-{HASH}version { return HASH_VERSION; }
-{HASH}line { return HASH_LINE; }
 
 {IDENTIFIER} {
-    yylval->sval = new std::string(yytext, yyleng);
-    return IDENTIFIER;
+    yylval->assign(yytext, yyleng);
+    return pp::Token::IDENTIFIER;
 }
 
 {DECIMAL_CONSTANT}|{OCTAL_CONSTANT}|{HEXADECIMAL_CONSTANT} {
-    yylval->sval = new std::string(yytext, yyleng);
-    return INT_CONSTANT;
+    yylval->assign(yytext, yyleng);
+    return pp::Token::CONST_INT;
 }
 
 ({DIGIT}+{EXPONENT_PART})|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?) {
-    yylval->sval = new std::string(yytext, yyleng);
-    return FLOAT_CONSTANT;
+    yylval->assign(yytext, yyleng);
+    return pp::Token::CONST_FLOAT;
 }
 
 {PUNCTUATOR} { return yytext[0]; }
@@ -125,45 +98,17 @@ int readInput(char* buf, int maxSize, yyscan_t scanner)
            (input->error() == pp::Input::kErrorNone) &&
            (nread == YY_NULL))
     {
-        int line = 0, file = 0;
-        pp::Token::decodeLocation(yyget_lineno(scanner), &line, &file);
-        file = input->stringIndex();
-        yyset_lineno(pp::Token::encodeLocation(line, file), scanner);
-
         nread = input->read(buf, maxSize);
-
-        if (input->error() == pp::Input::kErrorUnexpectedEOF)
-        {
-            // TODO(alokp): Report error.
-        }
     }
     return nread;
 }
 
-std::string* extractMacroName(const char* str, int len)
-{
-    // The input string is of the form {HASH}define{HSPACE}+{IDENTIFIER}
-    // We just need to find the last HSPACE.
-    ASSERT(str && (len > 8));  // strlen("#define ") == 8;
-
-    std::string* name = NULL;
-    for (int i = len - 1; i >= 0; --i)
-    {
-        if ((str[i] == ' ') || (str[i] == '\t'))
-        {
-            name = new std::string(str + i + 1, len - i - 1);
-            break;
-        }
-    }
-    ASSERT(name);
-    return name;
-}
-
 namespace pp {
 
-int Lexer::lex(YYSTYPE* lvalp, YYLTYPE* llocp)
+int Lexer::lex(Token* token)
 {
-    return yylex(lvalp, llocp, mHandle);
+    token->type = yylex(&token->value, &token->location, mHandle);
+    return token->type;
 }
 
 bool Lexer::initLexer()