Modified the interface of Preprocessor. Added it to the build file. Lexer…

src/build_angle.gyp

@@ -21,6 +21,8 @@
         'compiler/preprocessor/new/Lexer.cpp',
         'compiler/preprocessor/new/Lexer.h',
         'compiler/preprocessor/new/pp_lex.cpp',
+        'compiler/preprocessor/new/Preprocessor.cpp',
+        'compiler/preprocessor/new/Preprocessor.h',
         'compiler/preprocessor/new/Token.cpp',
         'compiler/preprocessor/new/Token.h',
       ],

src/common/version.h

 #define MAJOR_VERSION 1
 #define MINOR_VERSION 0
 #define BUILD_VERSION 0
-#define BUILD_REVISION 1032
+#define BUILD_REVISION 1033
 
 #define STRINGIFY(x) #x
 #define MACRO_STRINGIFY(x) STRINGIFY(x)

src/compiler/preprocessor/new/Lexer.cpp

@@ -6,12 +6,14 @@
 
 #include "Lexer.h"
 
+#include <cassert>
+
 #include "Input.h"
 
 namespace pp
 {
 
-Lexer::Lexer() : mHandle(0)
+Lexer::Lexer() : mHandle(0), mLeadingSpace(false)
 {
 }
 
@@ -22,6 +24,10 @@ Lexer::~Lexer()
 
 bool Lexer::init(int count, const char* const string[], const int length[])
 {
+    assert((count >=0) && (string != NULL));
+    if ((count < 0) || (string == NULL))
+        return false;
+
     mInput.reset(new Input(count, string, length));
     return initLexer();
 }

src/compiler/preprocessor/new/Lexer.h

@@ -33,6 +33,7 @@ class Lexer
     void destroyLexer();
 
     void* mHandle;  // Lexer handle.
+    bool mLeadingSpace;
     std::auto_ptr<Input> mInput;  // Input buffer.
 };
 

src/compiler/preprocessor/new/Preprocessor.cpp

@@ -6,41 +6,19 @@
 
 #include "Preprocessor.h"
 
-#include <algorithm>
-
-#include "compiler/debug.h"
-#include "Context.h"
-#include "stl_utils.h"
-
 namespace pp
 {
 
-Preprocessor::Preprocessor() : mContext(new Context)
-{
-}
-
-Preprocessor::~Preprocessor()
-{
-    delete mContext;
-}
-
-bool Preprocessor::process(int count,
-                           const char* const string[],
-                           const int length[])
+bool Preprocessor::init(int count,
+                        const char* const string[],
+                        const int length[])
 {
-    ASSERT((count >=0) && (string != NULL));
-    if ((count < 0) || (string == NULL))
-        return false;
-
-    reset();
-
-    return mContext->process(count, string, length, &mTokens);
+    return mLexer.init(count, string, length);
 }
 
-void Preprocessor::reset()
+int Preprocessor::lex(Token* token)
 {
-    std::for_each(mTokens.begin(), mTokens.end(), Delete());
-    mTokens.clear();
+    return mLexer.lex(token);
 }
 
 }  // namespace pp

src/compiler/preprocessor/new/Preprocessor.h

@@ -7,32 +7,24 @@
 #ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
 #define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
 
-#include "common/angleutils.h"
+#include "Lexer.h"
+#include "pp_utils.h"
 #include "Token.h"
 
 namespace pp
 {
 
-class Context;
-
 class Preprocessor
 {
   public:
-    Preprocessor();
-    ~Preprocessor();
+    Preprocessor() { }
 
-    bool process(int count, const char* const string[], const int length[]);
-    TokenIterator begin() const { return mTokens.begin(); }
-    TokenIterator end() const { return mTokens.end(); }
+    bool init(int count, const char* const string[], const int length[]);
+    int lex(Token* token);
 
   private:
-    DISALLOW_COPY_AND_ASSIGN(Preprocessor);
-
-    // Reset to initialized state.
-    void reset();
-
-    Context* mContext;
-    TokenVector mTokens;  // Output.
+    PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
+    Lexer mLexer;
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/pp.l

@@ -101,7 +101,7 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 "|="  {  return pp::Token::OP_OR_ASSIGN; }
 
 {PUNCTUATOR} { return yytext[0]; }
-[ \t\v\f]+   { /* Ignore whitespace */ }
+[ \t\v\f]+   { return ' '; }
 \n           { return yytext[0]; }
 
 <*><<EOF>>   { yyterminate(); }
@@ -126,6 +126,14 @@ namespace pp {
 int Lexer::lex(Token* token)
 {
     token->type = yylex(&token->value, &token->location, mHandle);
+    while (token->type == ' ')
+    {
+        mLeadingSpace = true;
+        token->type = yylex(&token->value, &token->location, mHandle);
+    }
+    token->setHasLeadingSpace(mLeadingSpace);
+    mLeadingSpace = false;
+
     return token->type;
 }
 

src/compiler/preprocessor/new/pp_lex.cpp

@@ -969,7 +969,7 @@ YY_RULE_SETUP
 	YY_BREAK
 case 27:
 YY_RULE_SETUP
-{ /* Ignore whitespace */ }
+{ return ' '; }
 	YY_BREAK
 case 28:
 /* rule 28 can match eol */
@@ -2143,6 +2143,14 @@ namespace pp {
 int Lexer::lex(Token* token)
 {
     token->type = pplex(&token->value,&token->location,mHandle);
+    while (token->type == ' ')
+    {
+        mLeadingSpace = true;
+        token->type = pplex(&token->value,&token->location,mHandle);
+    }
+    token->setHasLeadingSpace(mLeadingSpace);
+    mLeadingSpace = false;
+
     return token->type;
 }