Added an alternate lexer for the existing preprocessor. It is still behind a compile-time flag.

src/build_angle.gyp

@@ -73,6 +73,15 @@
         'compiler/ValidateLimitations.h',
         'compiler/VariableInfo.cpp',
         'compiler/VariableInfo.h',
+        # New Preprocessor
+        'compiler/preprocessor/new/Input.cpp',
+        'compiler/preprocessor/new/Input.h',
+        'compiler/preprocessor/new/Lexer.cpp',
+        'compiler/preprocessor/new/Lexer.h',
+        'compiler/preprocessor/new/pp_lex.cpp',
+        'compiler/preprocessor/new/Token.cpp',
+        'compiler/preprocessor/new/Token.h',
+        # Old preprocessor
         'compiler/preprocessor/atom.c',
         'compiler/preprocessor/atom.h',
         'compiler/preprocessor/compile.h',
@@ -80,6 +89,8 @@
         'compiler/preprocessor/cpp.h',
         'compiler/preprocessor/cppstruct.c',
         'compiler/preprocessor/length_limits.h',
+        'compiler/preprocessor/lexer_glue.cpp',
+        'compiler/preprocessor/lexer_glue.h',
         'compiler/preprocessor/memory.c',
         'compiler/preprocessor/memory.h',
         'compiler/preprocessor/parser.h',

src/compiler/ParseHelper.cpp

@@ -1490,8 +1490,9 @@ int PaParseStrings(int count, const char* const string[], const int length[],
     if (glslang_initialize(context))
         return 1;
 
-    glslang_scan(count, string, length, context);
-    int error = glslang_parse(context);
+    int error = glslang_scan(count, string, length, context);
+    if (!error)
+        error = glslang_parse(context);
 
     glslang_finalize(context);
     FinalizePreprocessor();

src/compiler/glslang.h

@@ -8,9 +8,9 @@ struct TParseContext;
 extern int glslang_initialize(TParseContext* context);
 extern int glslang_finalize(TParseContext* context);
 
-extern void glslang_scan(int count,
-                         const char* const string[],
-                         const int length[],
-                         TParseContext* context);
+extern int glslang_scan(int count,
+                        const char* const string[],
+                        const int length[],
+                        TParseContext* context);
 extern int glslang_parse(TParseContext* context);
 

src/compiler/glslang.l

@@ -568,19 +568,15 @@ int glslang_finalize(TParseContext* context) {
     return yylex_destroy(scanner);
 }
 
-void glslang_scan(int count, const char* const string[], const int length[],
-                  TParseContext* context) {
+int glslang_scan(int count, const char* const string[], const int length[],
+                 TParseContext* context) {
     yyrestart(NULL, context->scanner);
     yyset_lineno(EncodeSourceLoc(0, 1), context->scanner);
     context->AfterEOF = false;
     
     // Init preprocessor.
     cpp->pC = context;
-    cpp->PaWhichStr = 0;
-    cpp->PaArgv     = string;
-    cpp->PaArgc     = count;
-    cpp->PaStrLen   = length;
     cpp->pastFirstStatement = 0;
-    ScanFromString(string[0]);
+    return InitScannerInput(cpp, count, string, length);
 }
 

src/compiler/glslang_lex.cpp

-#line 17 "./glslang.l"
+#line 17 "./compiler/glslang.l"
 //
 // Copyright (c) 2010 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -21,7 +21,7 @@
 
 
 
-#line 25 "./glslang_lex.cpp"
+#line 25 "./compiler/glslang_lex.cpp"
 
 #define  YY_INT_ALIGNED short int
 
@@ -3202,19 +3202,15 @@ int glslang_finalize(TParseContext* context) {
     return yylex_destroy(scanner);
 }
 
-void glslang_scan(int count, const char* const string[], const int length[],
-                  TParseContext* context) {
+int glslang_scan(int count, const char* const string[], const int length[],
+                 TParseContext* context) {
     yyrestart(NULL,context->scanner);
     yyset_lineno(EncodeSourceLoc(0, 1),context->scanner);
     context->AfterEOF = false;
     
     // Init preprocessor.
     cpp->pC = context;
-    cpp->PaWhichStr = 0;
-    cpp->PaArgv     = string;
-    cpp->PaArgc     = count;
-    cpp->PaStrLen   = length;
     cpp->pastFirstStatement = 0;
-    ScanFromString(string[0]);
+    return InitScannerInput(cpp, count, string, length);
 }
 

src/compiler/preprocessor/lexer_glue.cpp

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+extern "C" {
+#include "compiler/preprocessor/lexer_glue.h"
+#include "compiler/preprocessor/slglobals.h"
+#include "compiler/preprocessor/scanner.h"
+}
+#include "compiler/preprocessor/new/Lexer.h"
+
+struct InputSrcLexer
+{
+    InputSrc base;
+    pp::Lexer* lexer;
+};
+
+static bool CopySymbolName(const std::string& name, yystypepp* yylvalpp)
+{
+    if (name.length() > MAX_SYMBOL_NAME_LEN)
+    {
+        CPPErrorToInfoLog("BUFFER OVERFLOW");
+        return false;
+    }
+    strcpy(yylvalpp->symbol_name, name.c_str());
+    return true;
+}
+
+static int lex(InputSrc* in, yystypepp* yylvalpp)
+{
+    InputSrcLexer* src = ((InputSrcLexer *)in);
+
+    pp::Token token;
+    int ret = src->lexer->lex(&token);
+    switch (ret)
+    {
+    case 0:  // EOF
+        delete src->lexer;
+        free(src);
+        cpp->currentInput = 0;
+        ret = EOF;
+        break;
+    case pp::Token::IDENTIFIER:
+        if (CopySymbolName(token.value, yylvalpp))
+        {
+            yylvalpp->sc_ident = LookUpAddString(atable, token.value.c_str());
+        }
+        ret = CPP_IDENTIFIER;
+        break;
+    case pp::Token::CONST_INT:
+        if (CopySymbolName(token.value, yylvalpp))
+        {
+            yylvalpp->sc_int = atoi(token.value.c_str());
+        }
+        ret = CPP_INTCONSTANT;
+        break;
+    case pp::Token::CONST_FLOAT:
+        CopySymbolName(token.value, yylvalpp);
+        ret = CPP_FLOATCONSTANT;
+        break;
+    case pp::Token::OP_INC:
+        ret = CPP_INC_OP;
+        break;
+    case pp::Token::OP_DEC:
+        ret = CPP_DEC_OP;
+        break;
+    case pp::Token::OP_RIGHT:
+        ret = CPP_RIGHT_OP;
+        break;
+    case pp::Token::OP_LE:
+        ret = CPP_LE_OP;
+        break;
+    case pp::Token::OP_GE:
+        ret = CPP_GE_OP;
+        break;
+    case pp::Token::OP_EQ:
+        ret = CPP_EQ_OP;
+        break;
+    case pp::Token::OP_NE:
+        ret = CPP_NE_OP;
+        break;
+    case pp::Token::OP_AND:
+        ret = CPP_AND_OP;
+        break;
+    case pp::Token::OP_XOR:
+        ret = CPP_XOR_OP;
+        break;
+    case pp::Token::OP_OR:
+        ret = CPP_OR_OP;
+        break;
+    case pp::Token::OP_ADD_ASSIGN:
+        ret = CPP_ADD_ASSIGN;
+        break;
+    case pp::Token::OP_SUB_ASSIGN:
+        ret = CPP_SUB_ASSIGN;
+        break;
+    case pp::Token::OP_MUL_ASSIGN:
+        ret = CPP_MUL_ASSIGN;
+        break;
+    case pp::Token::OP_DIV_ASSIGN:
+        ret = CPP_DIV_ASSIGN;
+        break;
+    case pp::Token::OP_MOD_ASSIGN:
+        ret = CPP_MOD_ASSIGN;
+        break;
+    case pp::Token::OP_LEFT_ASSIGN:
+        ret = CPP_LEFT_ASSIGN;
+        break;
+    case pp::Token::OP_RIGHT_ASSIGN:
+        ret = CPP_RIGHT_ASSIGN;
+        break;
+    case pp::Token::OP_AND_ASSIGN:
+        ret = CPP_AND_ASSIGN;
+        break;
+    case pp::Token::OP_XOR_ASSIGN:
+        ret = CPP_XOR_ASSIGN;
+        break;
+    case pp::Token::OP_OR_ASSIGN:
+        ret = CPP_OR_ASSIGN;
+        break;
+    default:
+        break;
+    }
+    SetLineNumber(token.location.line);
+    SetStringNumber(token.location.string);
+    return ret;
+}
+
+InputSrc* LexerInputSrc(int count, const char* const string[], const int length[])
+{
+    pp::Lexer* lexer = new pp::Lexer;
+    if (!lexer->init(count, string, length))
+    {
+        delete lexer;
+        return 0;
+    }
+
+    InputSrcLexer* in = (InputSrcLexer *) malloc(sizeof(InputSrcLexer));
+    memset(in, 0, sizeof(InputSrcLexer));
+    in->base.line = 1;
+    in->base.scan = lex;
+    in->lexer = lexer;
+
+    return &in->base;
+}

src/compiler/preprocessor/lexer_glue.h

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_LEXER_GLUE_H_
+#define COMPILER_PREPROCESSOR_LEXER_GLUE_H_
+
+struct InputSrc;
+
+InputSrc* LexerInputSrc(int count, const char* const string[], const int length[]);
+
+#endif  // COMPILER_PREPROCESSOR_LEXER_GLUE_H_
+

src/compiler/preprocessor/preprocess.h

@@ -47,5 +47,4 @@ extern CPPStruct *cpp;
 int InitCPPStruct(void);
 int InitScanner(CPPStruct *cpp);
 int InitAtomTable(AtomTable *atable, int htsize);
-int ScanFromString(const char *s);
 char* GetStringOfAtom(AtomTable *atable, int atom);

src/compiler/preprocessor/scanner.c

@@ -59,6 +59,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #include "compiler/preprocessor/slglobals.h"
+#include "compiler/preprocessor/lexer_glue.h"
 #include "compiler/util.h"
 
 typedef struct StringInputSrc {
@@ -66,6 +67,8 @@ typedef struct StringInputSrc {
     char *p;
 } StringInputSrc;
 
+static int ScanFromString(const char *s);
+
 static int eof_scan(InputSrc *is, yystypepp * yylvalpp)
 {
     return EOF;
@@ -126,6 +129,25 @@ int FreeScanner(void)
     return (FreeCPP());
 }
 
+// Define this to 1 to use the new lexer.
+#define CPP_USE_NEW_LEXER 0
+
+int InitScannerInput(CPPStruct *cpp, int count, const char* const string[], const int length[])
+{
+#if CPP_USE_NEW_LEXER
+    InputSrc* in = LexerInputSrc(count, string, length);
+    if (!in) return 1;
+    cpp->currentInput = in;
+#else
+    cpp->PaWhichStr = 0;
+    cpp->PaArgv     = string;
+    cpp->PaArgc     = count;
+    cpp->PaStrLen   = length;
+    ScanFromString(string[0]);
+#endif
+    return 0;
+}
+
 /*
  * str_getch()
  * takes care of reading from multiple strings.

src/compiler/preprocessor/scanner.h

@@ -69,7 +69,7 @@ typedef struct InputSrc {
 } InputSrc;
 
 int InitScanner(CPPStruct *cpp);   // Intialise the cpp scanner. 
-int ScanFromString(const char *);      // Start scanning the input from the string mentioned.
+int InitScannerInput(CPPStruct *cpp, int count, const char* const string[], const int length[]);
 int check_EOF(int);              // check if we hit a EOF abruptly 
 void CPPErrorToInfoLog(char *);   // sticking the msg,line into the Shader's.Info.log
 void SetLineNumber(int);

src/compiler/preprocessor/tokens.h

@@ -48,6 +48,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #if !defined(__TOKENS_H)
 #define __TOKENS_H 1
 
+#include <stdio.h>
 #include "compiler/preprocessor/parser.h"
 
 #define EOF_SY (-1)