Introduced preprocessing token types. This fixes a bug where invalid tokens…

src/compiler/preprocessor/new/DirectiveParser.cpp

@@ -221,7 +221,7 @@ void DirectiveParser::lex(Token* token)
     {
         mTokenizer->lex(token);
 
-        if (token->type == '#')
+        if (token->type == Token::PP_HASH)
         {
             parseDirective(token);
         }
@@ -242,7 +242,7 @@ void DirectiveParser::lex(Token* token)
 
 void DirectiveParser::parseDirective(Token* token)
 {
-    assert(token->type == '#');
+    assert(token->type == Token::PP_HASH);
 
     mTokenizer->lex(token);
     DirectiveType directive = getDirective(token);

src/compiler/preprocessor/new/Preprocessor.cpp

@@ -6,8 +6,10 @@
 
 #include "Preprocessor.h"
 
+#include <cassert>
 #include <sstream>
 
+#include "Diagnostics.h"
 #include "DirectiveParser.h"
 #include "Macro.h"
 #include "MacroExpander.h"
@@ -19,16 +21,18 @@ namespace pp
 
 struct PreprocessorImpl
 {
+    Diagnostics* diagnostics;
     MacroSet macroSet;
     Tokenizer tokenizer;
     DirectiveParser directiveParser;
     MacroExpander macroExpander;
 
-    PreprocessorImpl(Diagnostics* diagnostics,
+    PreprocessorImpl(Diagnostics* diag,
                      DirectiveHandler* directiveHandler) :
-        tokenizer(diagnostics),
-        directiveParser(&tokenizer, &macroSet, diagnostics, directiveHandler),
-        macroExpander(&directiveParser, &macroSet, diagnostics)
+        diagnostics(diag),
+        tokenizer(diag),
+        directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
+        macroExpander(&directiveParser, &macroSet, diag)
     {
     }
 };
@@ -79,7 +83,31 @@ void Preprocessor::predefineMacro(const char* name, int value)
 
 void Preprocessor::lex(Token* token)
 {
-    mImpl->macroExpander.lex(token);
+    bool validToken = false;
+    while (!validToken)
+    {
+        mImpl->macroExpander.lex(token);
+        switch (token->type)
+        {
+          // We should not be returning internal preprocessing tokens.
+          // Convert preprocessing tokens to compiler tokens or report
+          // diagnostics.
+          case Token::PP_HASH:
+            assert(false);
+            break;
+          case Token::PP_NUMBER:
+            mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
+                                       token->location, token->value);
+            break;
+          case Token::PP_OTHER:
+            mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
+                                       token->location, token->value);
+            break;
+          default:
+            validToken = true;
+            break;
+        }
+    }
 }
 
 }  // namespace pp

src/compiler/preprocessor/new/Token.h

@@ -46,7 +46,14 @@ struct Token
         OP_RIGHT_ASSIGN,
         OP_AND_ASSIGN,
         OP_XOR_ASSIGN,
-        OP_OR_ASSIGN
+        OP_OR_ASSIGN,
+
+        // Preprocessing token types.
+        // These types are used by the preprocessor internally.
+        // Preprocessor clients must not depend or check for them.
+        PP_HASH,
+        PP_NUMBER,
+        PP_OTHER
     };
     enum Flags
     {

src/compiler/preprocessor/new/Tokenizer.cpp

@@ -898,14 +898,8 @@ case 7:
 YY_RULE_SETUP
 {
     // # is only valid at start of line for preprocessor directives.
-    if (yyextra->lineStart) {
-        yylval->assign(1, yytext[0]);
-        return yytext[0];
-    } else {
-        yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
-                                     pp::SourceLocation(yyfileno, yylineno),
-                                     std::string(yytext, yyleng));
-    }
+    yylval->assign(1, yytext[0]);
+    return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
 }
 	YY_BREAK
 case 8:
@@ -934,9 +928,8 @@ YY_RULE_SETUP
 case 11:
 YY_RULE_SETUP
 {
-    yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
-                                 pp::SourceLocation(yyfileno, yylineno),
-                                 std::string(yytext, yyleng));
+    yylval->assign(yytext, yyleng);
+    return pp::Token::PP_NUMBER;
 }
 	YY_BREAK
 case 12:
@@ -1109,9 +1102,8 @@ YY_RULE_SETUP
 case 36:
 YY_RULE_SETUP
 {
-    yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
-                                 pp::SourceLocation(yyfileno, yylineno),
-                                 std::string(yytext, yyleng));
+    yylval->assign(1, yytext[0]);
+    return pp::Token::PP_OTHER;
 }
 	YY_BREAK
 case YY_STATE_EOF(INITIAL):

src/compiler/preprocessor/new/Tokenizer.l

@@ -100,14 +100,8 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 # {
     // # is only valid at start of line for preprocessor directives.
-    if (yyextra->lineStart) {
-        yylval->assign(1, yytext[0]);
-        return yytext[0];
-    } else {
-        yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
-                                     pp::SourceLocation(yyfileno, yylineno),
-                                     std::string(yytext, yyleng));
-    }
+    yylval->assign(1, yytext[0]);
+    return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
 }
 
 {IDENTIFIER} {
@@ -128,9 +122,8 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
     /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
     /* Rule to catch all invalid integers and floats. */
 ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
-    yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
-                                 pp::SourceLocation(yyfileno, yylineno),
-                                 std::string(yytext, yyleng));
+    yylval->assign(yytext, yyleng);
+    return pp::Token::PP_NUMBER;
 }
 
 "++" {
@@ -232,9 +225,8 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 }
 
 . {
-    yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
-                                 pp::SourceLocation(yyfileno, yylineno),
-                                 std::string(yytext, yyleng));
+    yylval->assign(1, yytext[0]);
+    return pp::Token::PP_OTHER;
 }
 
 <*><<EOF>> {