Hooked up the new preprocessor behind a compile-time flag ANGLE_USE_NEW_PREPROCESSOR.

src/build_angle.gyp

@@ -7,6 +7,7 @@
     'defines': [
       'ANGLE_DISABLE_TRACE',
       'ANGLE_COMPILE_OPTIMIZATION_LEVEL=D3DCOMPILE_OPTIMIZATION_LEVEL0',
+      'ANGLE_USE_NEW_PREPROCESSOR=0',
     ],
   },
   'targets': [
@@ -32,6 +33,7 @@
         'compiler/preprocessor/new/Macro.h',
         'compiler/preprocessor/new/MacroExpander.cpp',
         'compiler/preprocessor/new/MacroExpander.h',
+        'compiler/preprocessor/new/pp_utils.h',
         'compiler/preprocessor/new/Preprocessor.cpp',
         'compiler/preprocessor/new/Preprocessor.h',
         'compiler/preprocessor/new/SourceLocation.h',

src/compiler/DirectiveHandler.cpp

@@ -27,7 +27,7 @@ static TBehavior getBehavior(const std::string& str)
 
 TDirectiveHandler::TDirectiveHandler(TExtensionBehavior& extBehavior,
                                      TDiagnostics& diagnostics)
-    : mExtBehavior(extBehavior),
+    : mExtensionBehavior(extBehavior),
       mDiagnostics(diagnostics)
 {
 }
@@ -113,15 +113,15 @@ void TDirectiveHandler::handleExtension(const pp::SourceLocation& loc,
         }
         else
         {
-            for (TExtensionBehavior::iterator iter = mExtBehavior.begin();
-                 iter != mExtBehavior.end(); ++iter)
+            for (TExtensionBehavior::iterator iter = mExtensionBehavior.begin();
+                 iter != mExtensionBehavior.end(); ++iter)
                 iter->second = behaviorVal;
         }
         return;
     }
 
-    TExtensionBehavior::iterator iter = mExtBehavior.find(name);
-    if (iter != mExtBehavior.end())
+    TExtensionBehavior::iterator iter = mExtensionBehavior.find(name);
+    if (iter != mExtensionBehavior.end())
     {
         iter->second = behaviorVal;
         return;

src/compiler/DirectiveHandler.h

@@ -21,7 +21,7 @@ class TDirectiveHandler : public pp::DirectiveHandler
     virtual ~TDirectiveHandler();
 
     const TPragma& pragma() const { return mPragma; }
-    const TExtensionBehavior& extBehavior() const { return mExtBehavior; }
+    const TExtensionBehavior& extensionBehavior() const { return mExtensionBehavior; }
 
     virtual void handleError(const pp::SourceLocation& loc,
                              const std::string& msg);
@@ -39,7 +39,7 @@ class TDirectiveHandler : public pp::DirectiveHandler
 
   private:
     TPragma mPragma;
-    TExtensionBehavior& mExtBehavior;
+    TExtensionBehavior& mExtensionBehavior;
     TDiagnostics& mDiagnostics;
 };
 

src/compiler/ParseHelper.cpp

@@ -12,40 +12,6 @@
 #include "compiler/glslang.h"
 #include "compiler/preprocessor/new/SourceLocation.h"
 
-extern "C" {
-extern int InitPreprocessor();
-extern int FinalizePreprocessor();
-extern void PredefineIntMacro(const char *name, int value);
-}
-
-static void DefineExtensionMacros(const TExtensionBehavior& extBehavior)
-{
-    for (TExtensionBehavior::const_iterator iter = extBehavior.begin();
-         iter != extBehavior.end(); ++iter) {
-        PredefineIntMacro(iter->first.c_str(), 1);
-    }
-}
-
-///////////////////////////////////////////////////////////////////////
-//
-// Preprocessor
-//
-////////////////////////////////////////////////////////////////////////
-
-bool TParseContext::initPreprocessor()
-{
-    if (InitPreprocessor())
-        return false;
-
-    DefineExtensionMacros(directiveHandler.extBehavior());
-    return true;
-}
-
-void TParseContext::destroyPreprocessor()
-{
-    FinalizePreprocessor();
-}
-
 ///////////////////////////////////////////////////////////////////////
 //
 // Sub- vector and matrix fields
@@ -942,7 +908,7 @@ bool TParseContext::paramErrorCheck(int line, TQualifier qualifier, TQualifier p
 
 bool TParseContext::extensionErrorCheck(int line, const TString& extension)
 {
-    const TExtensionBehavior& extBehavior = directiveHandler.extBehavior();
+    const TExtensionBehavior& extBehavior = extensionBehavior();
     TExtensionBehavior::const_iterator iter = extBehavior.find(extension.c_str());
     if (iter == extBehavior.end()) {
         error(line, "extension", extension.c_str(), "is not supported");
@@ -963,9 +929,9 @@ bool TParseContext::extensionErrorCheck(int line, const TString& extension)
 
 bool TParseContext::supportsExtension(const char* extension)
 {
-    const TExtensionBehavior& extBehavior = directiveHandler.extBehavior();
-    TExtensionBehavior::const_iterator iter = extBehavior.find(extension);
-    return (iter != extBehavior.end());
+    const TExtensionBehavior& extbehavior = extensionBehavior();
+    TExtensionBehavior::const_iterator iter = extbehavior.find(extension);
+    return (iter != extbehavior.end());
 }
 
 void TParseContext::handleExtensionDirective(int line, const char* extName, const char* behavior)
@@ -1513,9 +1479,6 @@ int PaParseStrings(int count, const char* const string[], const int length[],
     if ((count == 0) || (string == NULL))
         return 1;
 
-    if (!context->initPreprocessor())
-        return 1;
-
     if (glslang_initialize(context))
         return 1;
 
@@ -1523,8 +1486,7 @@ int PaParseStrings(int count, const char* const string[], const int length[],
     if (!error)
         error = glslang_parse(context);
 
-    glslang_finalize(context);    
-    context->destroyPreprocessor();
+    glslang_finalize(context);
 
     return (error == 0) && (context->numErrors == 0) ? 0 : 1;
 }

src/compiler/ParseHelper.h

@@ -9,6 +9,7 @@
 #include "compiler/Diagnostics.h"
 #include "compiler/DirectiveHandler.h"
 #include "compiler/localintermediate.h"
+#include "compiler/preprocessor/new/Preprocessor.h"
 #include "compiler/ShHandle.h"
 #include "compiler/SymbolTable.h"
 
@@ -27,8 +28,6 @@ struct TParseContext {
     TParseContext(TSymbolTable& symt, TExtensionBehavior& ext, TIntermediate& interm, ShShaderType type, ShShaderSpec spec, int options, bool checksPrecErrors, const char* sourcePath, TInfoSink& is) :
             intermediate(interm),
             symbolTable(symt),
-            diagnostics(is),
-            directiveHandler(ext, diagnostics),
             shaderType(type),
             shaderSpec(spec),
             compileOptions(options),
@@ -42,11 +41,12 @@ struct TParseContext {
             currentFunctionType(NULL),
             functionReturnsValue(false),
             checksPrecisionErrors(checksPrecErrors),
+            diagnostics(is),
+            directiveHandler(ext, diagnostics),
+            preprocessor(&diagnostics, &directiveHandler),
             scanner(NULL) {  }
     TIntermediate& intermediate; // to hold and build a parse tree
     TSymbolTable& symbolTable;   // symbol table that goes with the language currently being parsed
-    TDiagnostics diagnostics;
-    TDirectiveHandler directiveHandler;
     ShShaderType shaderType;              // vertex or fragment language (future: pack or unpack)
     ShShaderSpec shaderSpec;              // The language specification compiler conforms to - GLES2 or WebGL.
     int compileOptions;
@@ -62,11 +62,11 @@ struct TParseContext {
     bool checksPrecisionErrors;  // true if an error will be generated when a variable is declared without precision, explicit or implicit.
     TString HashErrMsg;
     bool AfterEOF;
+    TDiagnostics diagnostics;
+    TDirectiveHandler directiveHandler;
+    pp::Preprocessor preprocessor;
     void* scanner;
 
-    bool initPreprocessor();
-    void destroyPreprocessor();
-
     TInfoSink& infoSink() { return diagnostics.infoSink(); }
     void error(TSourceLoc loc, const char *reason, const char* token,
                const char* extraInfoFormat, ...);
@@ -103,6 +103,7 @@ struct TParseContext {
     bool paramErrorCheck(int line, TQualifier qualifier, TQualifier paramQualifier, TType* type);
     bool extensionErrorCheck(int line, const TString&);
 
+    const TExtensionBehavior& extensionBehavior() const { return directiveHandler.extensionBehavior(); }
     bool supportsExtension(const char* extension);
     void handleExtensionDirective(int line, const char* extName, const char* behavior);
 

src/compiler/glslang.l

@@ -38,6 +38,7 @@ WHICH GENERATES THE GLSL ES LEXER (glslang_lex.cpp).
 %{
 #include "compiler/glslang.h"
 #include "compiler/ParseHelper.h"
+#include "compiler/preprocessor/new/Token.h"
 #include "compiler/util.h"
 #include "glslang_tab.h"
 
@@ -271,10 +272,15 @@ O           [0-7]
 
 %%
 
+#if !ANGLE_USE_NEW_PREPROCESSOR
 extern "C" {
 // Preprocessor interface.
 #include "compiler/preprocessor/preprocess.h"
 
+extern int InitPreprocessor();
+extern int FinalizePreprocessor();
+extern void PredefineIntMacro(const char *name, int value);
+
 #define SETUP_CONTEXT(pp) \
     TParseContext* context = (TParseContext*) pp->pC; \
     struct yyguts_t* yyg = (struct yyguts_t*) context->scanner;
@@ -393,17 +399,27 @@ void updateExtensionBehavior(const char* extName, const char* behavior)
     context->handleExtensionDirective(yylineno, extName, behavior);
 }
 }  // extern "C"
+#endif  // !ANGLE_USE_NEW_PREPROCESSOR
 
 int string_input(char* buf, int max_size, yyscan_t yyscanner) {
-    int len;
-
-    if ((len = yylex_CPP(buf, max_size)) == 0)
-        return 0;
-    if (len >= max_size) 
-        YY_FATAL_ERROR("input buffer overflow, can't enlarge buffer because scanner uses REJECT");
-
-    buf[len] = ' ';
-    return len+1;
+    int len = 0;
+
+#if ANGLE_USE_NEW_PREPROCESSOR
+    pp::Token token;
+    yyget_extra(yyscanner)->preprocessor.lex(&token);
+    len = token.type == pp::Token::LAST ? 0 : token.value.size();
+    if ((len > 0) && (len < max_size))
+        memcpy(buf, token.value.c_str(), len);
+    yyset_lineno(EncodeSourceLoc(token.location.file, token.location.line), yyscanner);
+#else
+    len = yylex_CPP(buf, max_size);
+#endif  // ANGLE_USE_NEW_PREPROCESSOR
+
+    if (len >= max_size)
+        YY_FATAL_ERROR("Input buffer overflow");
+    else if (len > 0)
+        buf[len++] = ' ';
+    return len;
 }
 
 int check_type(yyscan_t yyscanner) {
@@ -455,7 +471,12 @@ int glslang_finalize(TParseContext* context) {
     if (scanner == NULL) return 0;
     
     context->scanner = NULL;
-    return yylex_destroy(scanner);
+    yylex_destroy(scanner);
+
+#if !ANGLE_USE_NEW_PREPROCESSOR
+    FinalizePreprocessor();
+#endif
+    return 0;
 }
 
 int glslang_scan(int count, const char* const string[], const int length[],
@@ -463,10 +484,30 @@ int glslang_scan(int count, const char* const string[], const int length[],
     yyrestart(NULL, context->scanner);
     yyset_lineno(EncodeSourceLoc(0, 1), context->scanner);
     context->AfterEOF = false;
-    
-    // Init preprocessor.
+
+    // Initialize preprocessor.
+#if ANGLE_USE_NEW_PREPROCESSOR
+    if (!context->preprocessor.init(count, string, length))
+        return 1;
+#else
+    if (InitPreprocessor())
+        return 1;
     cpp->pC = context;
     cpp->pastFirstStatement = 0;
-    return InitScannerInput(cpp, count, string, length);
+    if (InitScannerInput(cpp, count, string, length))
+        return 1;
+#endif  // ANGLE_USE_NEW_PREPROCESSOR
+
+    // Define extension macros.
+    const TExtensionBehavior& extBehavior = context->extensionBehavior();
+    for (TExtensionBehavior::const_iterator iter = extBehavior.begin();
+         iter != extBehavior.end(); ++iter) {
+#if ANGLE_USE_NEW_PREPROCESSOR
+        context->preprocessor.predefineMacro(iter->first, 1);
+#else
+        PredefineIntMacro(iter->first.c_str(), 1);
+#endif
+    }
+    return 0;
 }
 

src/compiler/glslang_lex.cpp

@@ -807,6 +807,7 @@ WHICH GENERATES THE GLSL ES LEXER (glslang_lex.cpp).
 
 #include "compiler/glslang.h"
 #include "compiler/ParseHelper.h"
+#include "compiler/preprocessor/new/Token.h"
 #include "compiler/util.h"
 #include "glslang_tab.h"
 
@@ -2947,10 +2948,15 @@ void yyfree (void * ptr , yyscan_t yyscanner)
 
 #define YYTABLES_NAME "yytables"
 
+#if !ANGLE_USE_NEW_PREPROCESSOR
 extern "C" {
 // Preprocessor interface.
 #include "compiler/preprocessor/preprocess.h"
 
+extern int InitPreprocessor();
+extern int FinalizePreprocessor();
+extern void PredefineIntMacro(const char *name, int value);
+
 #define SETUP_CONTEXT(pp) \
     TParseContext* context = (TParseContext*) pp->pC; \
     struct yyguts_t* yyg = (struct yyguts_t*) context->scanner;
@@ -3069,17 +3075,27 @@ void updateExtensionBehavior(const char* extName, const char* behavior)
     context->handleExtensionDirective(yylineno, extName, behavior);
 }
 }  // extern "C"
+#endif  // !ANGLE_USE_NEW_PREPROCESSOR
 
 int string_input(char* buf, int max_size, yyscan_t yyscanner) {
-    int len;
-
-    if ((len = yylex_CPP(buf, max_size)) == 0)
-        return 0;
-    if (len >= max_size) 
-        YY_FATAL_ERROR("input buffer overflow, can't enlarge buffer because scanner uses REJECT");
+    int len = 0;
+
+#if ANGLE_USE_NEW_PREPROCESSOR
+    pp::Token token;
+    yyget_extra(yyscanner)->preprocessor.lex(&token);
+    len = token.type == pp::Token::LAST ? 0 : token.value.size();
+    if ((len > 0) && (len < max_size))
+        memcpy(buf, token.value.c_str(), len);
+    yyset_lineno(EncodeSourceLoc(token.location.file, token.location.line),yyscanner);
+#else
+    len = yylex_CPP(buf, max_size);
+#endif  // ANGLE_USE_NEW_PREPROCESSOR
 
-    buf[len] = ' ';
-    return len+1;
+    if (len >= max_size)
+        YY_FATAL_ERROR("Input buffer overflow");
+    else if (len > 0)
+        buf[len++] = ' ';
+    return len;
 }
 
 int check_type(yyscan_t yyscanner) {
@@ -3131,7 +3147,12 @@ int glslang_finalize(TParseContext* context) {
     if (scanner == NULL) return 0;
     
     context->scanner = NULL;
-    return yylex_destroy(scanner);
+    yylex_destroy(scanner);
+
+#if !ANGLE_USE_NEW_PREPROCESSOR
+    FinalizePreprocessor();
+#endif
+    return 0;
 }
 
 int glslang_scan(int count, const char* const string[], const int length[],
@@ -3139,10 +3160,30 @@ int glslang_scan(int count, const char* const string[], const int length[],
     yyrestart(NULL,context->scanner);
     yyset_lineno(EncodeSourceLoc(0, 1),context->scanner);
     context->AfterEOF = false;
-    
-    // Init preprocessor.
+
+    // Initialize preprocessor.
+#if ANGLE_USE_NEW_PREPROCESSOR
+    if (!context->preprocessor.init(count, string, length))
+        return 1;
+#else
+    if (InitPreprocessor())
+        return 1;
     cpp->pC = context;
     cpp->pastFirstStatement = 0;
-    return InitScannerInput(cpp, count, string, length);
+    if (InitScannerInput(cpp, count, string, length))
+        return 1;
+#endif  // ANGLE_USE_NEW_PREPROCESSOR
+
+    // Define extension macros.
+    const TExtensionBehavior& extBehavior = context->extensionBehavior();
+    for (TExtensionBehavior::const_iterator iter = extBehavior.begin();
+         iter != extBehavior.end(); ++iter) {
+#if ANGLE_USE_NEW_PREPROCESSOR
+        context->preprocessor.predefineMacro(iter->first, 1);
+#else
+        PredefineIntMacro(iter->first.c_str(), 1);
+#endif
+    }
+    return 0;
 }
 

src/compiler/preprocessor/new/Preprocessor.cpp

@@ -8,17 +8,40 @@
 
 #include <sstream>
 
+#include "DirectiveParser.h"
+#include "Macro.h"
+#include "MacroExpander.h"
 #include "Token.h"
+#include "Tokenizer.h"
 
 namespace pp
 {
 
+struct PreprocessorImpl
+{
+    MacroSet macroSet;
+    Tokenizer tokenizer;
+    DirectiveParser directiveParser;
+    MacroExpander macroExpander;
+
+    PreprocessorImpl(Diagnostics* diagnostics,
+                     DirectiveHandler* directiveHandler) :
+        tokenizer(diagnostics),
+        directiveParser(&tokenizer, &macroSet, diagnostics, directiveHandler),
+        macroExpander(&directiveParser, &macroSet, diagnostics)
+    {
+    }
+};
+
 Preprocessor::Preprocessor(Diagnostics* diagnostics,
-                           DirectiveHandler* directiveHandler) :
-    mTokenizer(diagnostics),
-    mDirectiveParser(&mTokenizer, &mMacroSet, diagnostics, directiveHandler),
-    mMacroExpander(&mDirectiveParser, &mMacroSet, diagnostics)
+                           DirectiveHandler* directiveHandler)
+{
+    mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
+}
+
+Preprocessor::~Preprocessor()
 {
+    delete mImpl;
 }
 
 bool Preprocessor::init(int count,
@@ -33,10 +56,10 @@ bool Preprocessor::init(int count,
     predefineMacro("__VERSION__", kGLSLVersion);
     predefineMacro("GL_ES", 1);
 
-    return mTokenizer.init(count, string, length);
+    return mImpl->tokenizer.init(count, string, length);
 }
 
-void Preprocessor::predefineMacro(const std::string& name, int value)
+void Preprocessor::predefineMacro(const char* name, int value)
 {
     std::stringstream stream;
     stream << value;
@@ -51,12 +74,12 @@ void Preprocessor::predefineMacro(const std::string& name, int value)
     macro.name = name;
     macro.replacements.push_back(token);
 
-    mMacroSet[name] = macro;
+    mImpl->macroSet[name] = macro;
 }
 
 void Preprocessor::lex(Token* token)
 {
-    mMacroExpander.lex(token);
+    mImpl->macroExpander.lex(token);
 }
 
 }  // namespace pp

src/compiler/preprocessor/new/Preprocessor.h

@@ -7,21 +7,21 @@
 #ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
 #define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
 
-#include "DirectiveParser.h"
-#include "Macro.h"
-#include "MacroExpander.h"
-#include "Tokenizer.h"
+#include "pp_utils.h"
 
 namespace pp
 {
 
 class Diagnostics;
 class DirectiveHandler;
+struct PreprocessorImpl;
+struct Token;
 
 class Preprocessor
 {
   public:
     Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
+    ~Preprocessor();
 
     // count: specifies the number of elements in the string and length arrays.
     // string: specifies an array of pointers to strings.
@@ -34,17 +34,14 @@ class Preprocessor
     // is null terminated.
     bool init(int count, const char* const string[], const int length[]);
     // Adds a pre-defined macro.
-    void predefineMacro(const std::string& name, int value);
+    void predefineMacro(const char* name, int value);
 
     void lex(Token* token);
 
   private:
     PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
 
-    MacroSet mMacroSet;
-    Tokenizer mTokenizer;
-    DirectiveParser mDirectiveParser;
-    MacroExpander mMacroExpander;
+    PreprocessorImpl* mImpl;
 };
 
 }  // namespace pp

tests/preprocessor_tests/input_test.cpp

@@ -5,6 +5,7 @@
 //
 
 #include "PreprocessorTest.h"
+#include "Input.h"
 #include "Token.h"
 
 class InitTest : public PreprocessorTest