Fix several issues in the preprocessor:

Test/baseResults/cppComplexExpr.vert.out

 ERROR: 0:46: 'xyxwx' : illegal vector field selection 
 ERROR: 0:46: 'xyxwx' : illegal vector field selection 
-ERROR: 2 compilation errors.  No code generated.
+ERROR: 0:51: '' : missing #endif 
+ERROR: 3 compilation errors.  No code generated.
 
 ERROR: node is still EOpNull!
 0:4  Sequence

Test/baseResults/cppSimple.vert.out

+Warning, version 400 is not yet complete; some version-specific features are present, but many are missing.
 ERROR: 0:77: '#error' : good1  
 ERROR: 0:81: '#error' : good2  
 ERROR: 0:85: '#error' : good3  
@@ -6,20 +7,42 @@ ERROR: 0:93: '#error' : good5
 ERROR: 0:97: '#error' : good6  
 ERROR: 0:101: 'preprocessor' : expected ')' 
 ERROR: 0:101: '#error' : bad1  
-WARNING: 0:104: '#if' : unexpected tokens following #if directive - expected a newline 
+ERROR: 0:104: '#if' : unexpected tokens following directive 
 ERROR: 0:105: '#error' : bad2  
 ERROR: 0:109: 'preprocessor' : expected ')' 
 ERROR: 0:109: '#error' : bad3  
-WARNING: 0:112: '#if' : unexpected tokens following #if directive - expected a newline 
+ERROR: 0:112: '#if' : unexpected tokens following directive 
 ERROR: 0:113: '#error' : bad4  
 ERROR: 0:117: 'preprocessor' : expected ')' 
 ERROR: 0:117: '#error' : bad5  
-WARNING: 0:120: '#if' : unexpected tokens following #if directive - expected a newline 
+ERROR: 0:120: '#if' : unexpected tokens following directive 
 ERROR: 0:121: '#error' : bad6  
-ERROR: 0:133: '#' : preprocessor directive cannot be preceded by another token 
-INTERNAL ERROR: 0:133: Unknown PP token
-ERROR: 0:133: '' :  syntax error
-ERROR: 17 compilation errors.  No code generated.
+ERROR: 0:122: '#endif' : unexpected tokens following directive 
+ERROR: 0:135: '""' : string literals not supported 
+ERROR: 0:136: '""' : string literals not supported 
+ERROR: 0:136: 'length' : no matching overloaded function found 
+ERROR: 0:136: '=' :  cannot convert from 'const float' to 'int'
+ERROR: 0:138: ''' : character literals not supported 
+ERROR: 0:138: ''' : character literals not supported 
+ERROR: 0:141: '#define' : reserved built-in name prefix: GL_
+ERROR: 0:142: '#define' : reserved built-in name prefix: GL_
+ERROR: 0:143: '#define' : names containing consecutive underscores are reserved 
+ERROR: 0:144: '#define' : names containing consecutive underscores are reserved 
+ERROR: 0:145: '#define' : names containing consecutive underscores are reserved 
+ERROR: 0:148: '#else' : unexpected tokens following directive 
+ERROR: 0:149: '#else' : #elif after #else 
+ERROR: 0:155: '#else' : unexpected tokens following directive 
+ERROR: 0:158: '#else' : #else after #else 
+ERROR: 0:160: '#endif' : unexpected tokens following directive 
+ERROR: 0:164: '#define' : duplicate macro parameter 
+ERROR: 0:173: '#define' : Macro redefined; different number of arguments: m4
+ERROR: 0:178: '#define' : Macro redefined; different number of arguments: m5
+ERROR: 0:182: '#define' : Macro redefined; different number of arguments: m6
+ERROR: 0:185: '#define' : Macro redefined; different substitutions: m7
+ERROR: 0:192: '#define' : Macro redefined; different substitutions: m8
+ERROR: 0:196: '#define' : Macro redefined; different argument names: m9
+ERROR: 0:206: '' : missing #endif 
+ERROR: 43 compilation errors.  No code generated.
 
 ERROR: node is still EOpNull!
 0:5  Sequence
@@ -59,7 +82,10 @@ ERROR: node is still EOpNull!
 0:65        Constant:
 0:65          0.050000
 0:69      move second child to first child (4-component vector of float)
-0:69        'gl_Position' (gl_Position 4-component vector of float)
+0:69        gl_Position: direct index for structure (gl_Position 4-component vector of float)
+0:69          '__anon__1' (out block{gl_Position,gl_PointSize,gl_ClipDistance,gl_ClipVertex,gl_FrontColor,gl_BackColor,gl_FrontSecondaryColor,gl_BackSecondaryColor,gl_TexCoord,gl_FogFragCoord})
+0:69          Constant:
+0:69            0 (const uint)
 0:69        Construct vec4 (4-component vector of float)
 0:69          'sum' (float)
 0:124  Sequence
@@ -76,17 +102,31 @@ ERROR: node is still EOpNull!
 0:126    move second child to first child (int)
 0:126      'version' (int)
 0:126      Constant:
-0:126        110 (const int)
+0:126        400 (const int)
 0:130  Sequence
 0:130    move second child to first child (float)
 0:130      'twoPi' (float)
 0:130      Constant:
 0:130        6.280000
+0:199  Sequence
+0:199    move second child to first child (int)
+0:199      'n' (int)
+0:199      Constant:
+0:199        15 (const int)
+0:202  Sequence
+0:202    move second child to first child (double)
+0:202      'f' (double)
+0:202      Constant:
+0:202        0.000800
 0:?   Linker Objects
 0:?     'sum' (float)
 0:?     'linenumber' (int)
 0:?     'filenumber' (int)
 0:?     'version' (int)
 0:?     'twoPi' (float)
-0:?     'tod' (float)
+0:?     'a' (int)
+0:?     'n' (int)
+0:?     'f' (double)
+0:?     'gl_VertexID' (gl_VertexId int)
+0:?     'gl_InstanceID' (gl_InstanceId int)
 

Test/cppComplexExpr.vert

@@ -46,3 +46,6 @@ float foo()
     return ADD(gl_Position.xyxwx, 3.0)  // ERROR, should be this line number
     return ADD(gl_Position.y, 3.0)
 }
+
+#if 0
+// ERROR, EOF
\ No newline at end of file

Test/cppSimple.vert

-#version 110
+#version 400
 
 #define ON
 
@@ -119,7 +119,7 @@ sum += 0.05;
 
 #if ((AA == BB || (AA == CC))))
 #error bad6
-#endif
+#endif extra tokens
 
 int linenumber = __LINE__;
 int filenumber = __FILE__;
@@ -129,5 +129,78 @@ int version = __VERSION__;
 #define TWOPI (2.0 * PI)
 float twoPi = TWOPI;
 
-#define PASTE(a,b) a ## b
-float PASTE(tod, ay) = 17;
+//#define PASTE(a,b) a ## b
+//float PASTE(tod, ay) = 17;
+
+"boo" // ERROR
+int a = length("aoenatuh");  // ERROR
+
+'int';  // ERROR
+
+// ERROR: all the following are reserved
+#define GL_
+#define GL_Macro 1
+#define __M 
+#define M__
+#define ABC__DE abc
+
+#if 4
+#else extra
+#elif
+// ERROR elif after else
+#endif
+
+#if blah
+  #if 0
+  #else extra
+    #ifdef M
+    #else
+    #else
+    // ERROR else after else
+    #endif  extra
+  #endif
+#endif
+
+#define m1(a,a)  // ERROR
+#define m2(a,b)
+
+// okay
+#define m3 (a)
+#define m3 (a)
+
+// ERROR
+#define m4(b)
+#define m4 (b)
+
+// ERROR
+#define m5 (b)
+#define m5(b)
+
+// ERROR
+#define m6(a)
+#define m6
+
+// ERROR (whitespace)
+#define m7 (a)
+#define m7 ( a)
+
+#define m80(a,b) is + exactly m3 the same
+#define m80(a,b) is + exactly m3 the same
+
+// ERROR
+#define m8(a,b) almost + exactly m3 the same
+#define m8(a,b) almost + exactly m3 thee same
+
+// ERROR
+#define m9(a,b,c) aoe
+#define m9(a,d,c) aoe
+
+#define n1 0xf
+int n = n1;
+
+#define f1 .08e-2Lf
+double f = f1;
+
+#if 1
+#else
+// ERROR, missing #endif
\ No newline at end of file

glslang/MachineIndependent/Scan.cpp

@@ -480,6 +480,8 @@ int TScanContext::tokenize(TPpContext* pp, TParserToken& token)
     parserToken = &token;
     TPpToken ppToken;
     tokenText = pp->tokenize(&ppToken);
+    if (tokenText == 0)
+        return 0;
 
     loc = ppToken.loc;
     parserToken->sType.lex.loc = loc;

glslang/MachineIndependent/Scan.h

@@ -46,8 +46,8 @@ namespace glslang {
 //
 class TInputScanner {
 public:
-    TInputScanner(int n, const char* const s[], size_t L[], int b = 0) : 
-        numSources(n), sources(s), lengths(L), currentSource(0), currentChar(0), stringBias(b)
+    TInputScanner(int n, const char* const s[], size_t L[], int b = 0, int f = 0) : 
+        numSources(n), sources(s), lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f)
     {
         loc = new TSourceLoc[numSources];
         loc[currentSource].string = -stringBias;
@@ -105,7 +105,7 @@ public:
     void setLine(int newLine) { loc[currentSource].line = newLine; }
     void setString(int newString) { loc[currentSource].string = newString; }
 
-    const TSourceLoc& getSourceLoc() const { return loc[currentSource]; }
+    const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; }
 
     void consumeWhiteSpace(bool& foundNonSpaceTab);
     bool consumeComment();
@@ -147,6 +147,7 @@ protected:
     TSourceLoc* loc;  // an array
 
     int stringBias;   // the first string that is the user's string number 0
+    int finale;       // number of internal strings after user's last string
 };
 
 } // end namespace glslang

glslang/MachineIndependent/ShaderLang.cpp

@@ -499,7 +499,7 @@ bool CompileDeferred(
     lengths[0] = strlen(strings[0]);
     strings[numStrings + 1] = "\n int;";
     lengths[numStrings + 1] = strlen(strings[numStrings + 1]);
-    TInputScanner fullInput(numStrings + 2, strings, lengths, 1);
+    TInputScanner fullInput(numStrings + 2, strings, lengths, 1, 1);
 
     // Push a new symbol allocation scope that can for the shader's globals.
     symbolTable.push();

glslang/MachineIndependent/preprocessor/PpAtom.cpp

@@ -150,8 +150,12 @@ const char *TPpContext::GetAtomString(int atom)
         return "<null atom>";
     if (atom < 0)
         return "<EOF>";
-    if ((size_t)atom < stringMap.size())
+    if ((size_t)atom < stringMap.size()) {
+        if (stringMap[atom] == 0)
+            return "<invalid atom>";
+        else
             return stringMap[atom]->c_str();
+    }
 
     return "<invalid atom>";
 }
@@ -163,7 +167,7 @@ int TPpContext::AddAtomFixed(const char *s, int atom)
 {
     TAtomMap::const_iterator it = atomMap.insert(std::pair<TString, int>(s, atom)).first;
     if (stringMap.size() < (size_t)atom + 1)
-        stringMap.resize(atom + 100);
+        stringMap.resize(atom + 100, 0);
     stringMap[atom] = &it->first;
 
     return atom;

glslang/MachineIndependent/preprocessor/PpContext.cpp

@@ -106,9 +106,9 @@ void TPpContext::setInput(TInputScanner& input, bool versionWillBeError)
     StringInputSrc *in = (StringInputSrc *)malloc(sizeof(StringInputSrc));
     memset(in, 0, sizeof(StringInputSrc));
     in->input = &input;
-    in->base.scan = byte_scan;
-    in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))str_getch;
-    in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))str_ungetch;
+    in->base.scan = sourceScan;
+    in->base.getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))sourceGetCh;
+    in->base.ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))sourceUngetCh;
     in->base.prev = currentInput;
     currentInput = &in->base;
     errorOnVersion = versionWillBeError;

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -84,7 +84,21 @@ namespace glslang {
 
 class TPpToken {
 public:
-    TPpToken() { loc.line = 0; loc.string = 0; name[0] = 0; }
+    TPpToken() : token(0), ival(0), dval(0.0), atom(0)
+    { 
+        loc.line = 0; 
+        loc.string = 0; 
+        name[0] = 0;
+    }
+
+    bool operator==(const TPpToken& right)
+    {
+        return token == right.token && atom == right.atom &&
+               ival == right.ival && dval == right.dval &&
+               strcmp(name, right.name) == 0;
+    }
+    bool operator!=(const TPpToken& right) { return ! operator==(right); }
+
     static const int maxTokenLength = 1024;
 
     TSourceLoc loc;
@@ -110,6 +124,7 @@ public:
 
     const char* tokenize(TPpToken* ppToken);
 
+    // TODO: preprocessor simplification: this should be a base class, not a set of function pointers
     struct InputSrc {
         struct InputSrc	*prev;
         int			(*scan)(TPpContext*, struct InputSrc *, TPpToken *);
@@ -127,7 +142,6 @@ public:
 
     struct TokenStream {
         TokenStream *next;
-        char *name;
         TokenBlock *head;
         TokenBlock *current;
     };
@@ -231,6 +245,7 @@ protected:
     int CPPdefine(TPpToken * ppToken);
     int CPPundef(TPpToken * ppToken);
     int CPPelse(int matchelse, TPpToken * ppToken);
+    int extraTokenCheck(int atom, TPpToken* ppToken, int token);
     int eval(int token, int prec, int *res, int *err, TPpToken * ppToken);
     int CPPif (TPpToken * ppToken); 
     int CPPifdef(int defined, TPpToken * ppToken);
@@ -259,18 +274,16 @@ protected:
     //
     // From PpTokens.cpp
     //
-    char* idstr(const char *fstr, MemoryPool *pool);
     TPpContext::TokenBlock* lNewBlock(TokenStream *fTok, MemoryPool *pool);
     void lAddByte(TokenStream *fTok, unsigned char fVal);
     int lReadByte(TokenStream *pTok);
-    TokenStream *NewTokenStream(const char *name, MemoryPool *pool);
+    TokenStream *NewTokenStream(MemoryPool *pool);
     void DeleteTokenStream(TokenStream *pTok);
-    void RecordToken(TokenStream *pTok, int token, TPpToken * ppToken);
+    void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken);
     void RewindTokenStream(TokenStream *pTok);
-    int ReadToken(TokenStream *pTok, TPpToken * ppToken);
+    int ReadToken(TokenStream* pTok, TPpToken* ppToken);
     int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpContext *));
-    void UngetToken(int token, TPpToken * ppToken);
-    void DumpTokenStream(FILE *fp, TokenStream *s, TPpToken * ppToken);
+    void UngetToken(int token, TPpToken* ppToken);
     struct TokenInputSrc {
         InputSrc            base;
         TokenStream *tokens;
@@ -292,12 +305,12 @@ protected:
         TInputScanner* input;
     };
     int InitScanner(TPpContext *cpp);
-    static int str_getch(TPpContext*, StringInputSrc *in);
-    static void str_ungetch(TPpContext*, StringInputSrc *in, int ch, TPpToken *type);
+    static int sourceGetCh(TPpContext*, StringInputSrc *in);
+    static void sourceUngetCh(TPpContext*, StringInputSrc *in, int ch, TPpToken *type);
     int ScanFromString(char *s);
-    int check_EOF(int token);
+    bool check_EOF(int token);
     int lFloatConst(char *str, int len, int ch, TPpToken * ppToken);
-    static int byte_scan(TPpContext*, InputSrc *in, TPpToken * ppToken);
+    static int sourceScan(TPpContext*, InputSrc *in, TPpToken * ppToken);
 
     //
     // From PpAtom.cpp

glslang/MachineIndependent/preprocessor/PpScanner.cpp

@@ -105,7 +105,7 @@ int TPpContext::InitScanner(TPpContext *cpp)
     return 1;
 }
 
-int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
+int TPpContext::sourceGetCh(TPpContext* pp, StringInputSrc *in)
 {
     int ch = in->input->get();
 
@@ -115,7 +115,7 @@ int TPpContext::str_getch(TPpContext* pp, StringInputSrc *in)
     return ch;
 }
 
-void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
+void TPpContext::sourceUngetCh(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type)
 {
     in->input->unget();
 }
@@ -130,7 +130,7 @@ void TPpContext::str_ungetch(TPpContext* pp, StringInputSrc *in, int ch, TPpToke
 *         letter 'e', or a precision ending (e.g., F or LF).
 */
 
-int TPpContext::lFloatConst(char *str, int len, int ch, TPpToken * ppToken)
+int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken)
 {
     bool HasDecimalOrExponent = false;
     int declen, exp, ExpSign;
@@ -241,13 +241,12 @@ int TPpContext::lFloatConst(char *str, int len, int ch, TPpToken * ppToken)
         return CPP_DOUBLECONSTANT;
     else
         return CPP_FLOATCONSTANT;
-} // lFloatConst
-
-///////////////////////////////////////////////////////////////////////////////////////////////
-///////////////////////////////////////// Normal Scanner //////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////////////////////////
+}
 
-int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
+//
+// Scanner used to tokenize source stream.
+//
+int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken)
 {
     char tokenText[TPpToken::maxTokenLength + 1];
     int AlreadyComplained = 0;
@@ -268,8 +267,6 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
         switch (ch) {
         default:
             return ch; // Single character token
-        case EOF:
-            return EOF;
         case 'A': case 'B': case 'C': case 'D': case 'E':
         case 'F': case 'G': case 'H': case 'I': case 'J':
         case 'K': case 'L': case 'M': case 'N': case 'O':
@@ -704,14 +701,17 @@ int TPpContext::byte_scan(TPpContext* pp, InputSrc *in, TPpToken * ppToken)
             }
         }
     }
-} // byte_scan
+}
 
+//
+// Return string pointer to next token.
+// Return 0 when no more tokens.
+//
 const char* TPpContext::tokenize(TPpToken* ppToken)
 {    
     int token = '\n';
 
     for(;;) {
-
         const char* tokenString = 0;
         token = currentInput->scan(this, currentInput, ppToken);
         ppToken->token = token;
@@ -742,7 +742,13 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
         else if (token == CPP_INTCONSTANT || token == CPP_UINTCONSTANT ||
                  token == CPP_FLOATCONSTANT || token == CPP_DOUBLECONSTANT)
             tokenString = ppToken->name;
-        else
+        else if (token == CPP_STRCONSTANT) {
+            parseContext.error(ppToken->loc, "string literals not supported", "\"\"", "");
+            tokenString = 0;
+        } else if (token == '\'') {
+            parseContext.error(ppToken->loc, "character literals not supported", "\'", "");
+            tokenString = 0;
+        } else
             tokenString = GetAtomString(token);
 
         if (tokenString) {
@@ -752,19 +758,19 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
             return tokenString;
         }
     }
-
-    return 0;
-} // PpTokenize
+}
 
 //Checks if the token just read is EOF or not.
-int TPpContext::check_EOF(int token)
+bool TPpContext::check_EOF(int token)
 {
     if (token == EOF) {
         if (ifdepth > 0)
-            parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "#if", "");
-        return 1;
+            parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "", "");
+
+        return true;
     }
-    return 0;
+
+    return false;
 }
 
 } // end namespace glslang

glslang/MachineIndependent/preprocessor/PpTokens.h

@@ -99,7 +99,6 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CPP_OR_OP          276
 #define CPP_INC_OP         277
 #define CPP_STRCONSTANT    278
-#define CPP_TYPEIDENTIFIER 279
 #define CPP_RIGHT_ASSIGN   280
 #define CPP_LEFT_ASSIGN    281
 #define CPP_AND_ASSIGN     282