Moved error-handling to a separate class - Diagnostics. We were earlier…

src/build_angle.gyp

@@ -16,6 +16,8 @@
       'include_dirs': [
       ],
       'sources': [
+        'compiler/preprocessor/new/Diagnostics.cpp',
+        'compiler/preprocessor/new/Diagnostics.h',
         'compiler/preprocessor/new/DirectiveParser.cpp',
         'compiler/preprocessor/new/DirectiveParser.h',
         'compiler/preprocessor/new/ExpressionParser.cpp',
@@ -27,6 +29,7 @@
         'compiler/preprocessor/new/MacroExpander.h',
         'compiler/preprocessor/new/Preprocessor.cpp',
         'compiler/preprocessor/new/Preprocessor.h',
+        'compiler/preprocessor/new/SourceLocation.h',
         'compiler/preprocessor/new/Token.cpp',
         'compiler/preprocessor/new/Token.h',
         'compiler/preprocessor/new/Tokenizer.cpp',

src/compiler/preprocessor/new/Diagnostics.cpp

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "Diagnostics.h"
+
+#include <cassert>
+
+namespace pp
+{
+
+void Diagnostics::report(ID id,
+                         const SourceLocation& loc,
+                         const std::string& text)
+{
+    // TODO(alokp): Keep a count of errors and warnings.
+    print(id, loc, text);
+}
+
+Diagnostics::Severity Diagnostics::severity(ID id)
+{
+    if ((id > ERROR_BEGIN) && (id < ERROR_END))
+        return ERROR;
+
+    if ((id > WARNING_BEGIN) && (id < WARNING_END))
+        return WARNING;
+
+    assert(false);
+    return ERROR;
+}
+
+}  // namespace pp

src/compiler/preprocessor/new/Diagnostics.h

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
+#define COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
+
+#include <string>
+
+namespace pp
+{
+
+struct SourceLocation;
+
+// Base class for reporting diagnostic messages.
+// Derived classes are responsible for formatting and printing the messages.
+class Diagnostics
+{
+  public:
+    enum ID
+    {
+        ERROR_BEGIN,
+        INTERNAL_ERROR,
+        OUT_OF_MEMORY,
+        INVALID_CHARACTER,
+        INVALID_NUMBER,
+        INVALID_DIRECTIVE,
+        INVALID_EXPRESSION,
+        DIVISION_BY_ZERO,
+        EOF_IN_COMMENT,
+        EOF_IN_DIRECTIVE,
+        UNEXPECTED_TOKEN_IN_DIRECTIVE,
+        ERROR_END,
+
+        WARNING_BEGIN,
+        WARNING_END
+    };
+
+    void report(ID id, const SourceLocation& loc, const std::string& text);
+
+  protected:
+    enum Severity
+    {
+        ERROR,
+        WARNING
+    };
+    Severity severity(ID id);
+
+    virtual void print(ID id,
+                       const SourceLocation& loc,
+                       const std::string& text) = 0;
+};
+
+}  // namespace pp
+#endif  // COMPILER_PREPROCESSOR_DIAGNOSTICS_H_

src/compiler/preprocessor/new/DirectiveParser.cpp

@@ -8,6 +8,7 @@
 
 #include <cassert>
 
+#include "Diagnostics.h"
 #include "ExpressionParser.h"
 #include "MacroExpander.h"
 #include "Token.h"
@@ -48,6 +49,13 @@ class DefinedParser : public Lexer
     Lexer* mLexer;
 };
 
+DirectiveParser::DirectiveParser(Tokenizer* tokenizer,
+                                 Diagnostics* diagnostics) :
+    mTokenizer(tokenizer),
+    mDiagnostics(diagnostics)
+{
+}
+
 void DirectiveParser::lex(Token* token)
 {
     do
@@ -91,16 +99,24 @@ void DirectiveParser::parseDirective(Token* token)
         else if (token->value == kDirectiveLine)
             parseLine(token);
         else
-            token->type = pp::Token::INVALID_DIRECTIVE;
+            mDiagnostics->report(Diagnostics::INVALID_DIRECTIVE,
+                                 token->location,
+                                 token->value.c_str());
     }
 
+    if ((token->type != '\n') && (token->type != 0))
+        mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN_IN_DIRECTIVE,
+                             token->location,
+                             token->value.c_str());
+
     while (token->type != '\n')
     {
         if (token->type == 0) {
-            //token->type = pp::Token::EOF_IN_DIRECTIVE;
+            mDiagnostics->report(Diagnostics::EOF_IN_DIRECTIVE,
+                                 token->location,
+                                 token->value.c_str());
             break;
         }
-        //token->type = pp::Token::INVALID_DIRECTIVE;
         mTokenizer->lex(token);
     }
 }
@@ -125,8 +141,8 @@ void DirectiveParser::parseIf(Token* token)
     assert(token->value == kDirectiveIf);
 
     DefinedParser definedParser(mTokenizer);
-    MacroExpander macroExpander(&definedParser);
-    ExpressionParser expressionParser(&macroExpander);
+    MacroExpander macroExpander(&definedParser, mDiagnostics);
+    ExpressionParser expressionParser(&macroExpander, mDiagnostics);
     macroExpander.lex(token);
 
     int expression = 0;
@@ -207,7 +223,7 @@ void DirectiveParser::parseLine(Token* token)
 {
     // TODO(alokp): Implement me.
     assert(token->value == kDirectiveLine);
-    MacroExpander macroExpander(mTokenizer);
+    MacroExpander macroExpander(mTokenizer, mDiagnostics);
     macroExpander.lex(token);
 }
 

src/compiler/preprocessor/new/DirectiveParser.h

@@ -13,12 +13,13 @@
 namespace pp
 {
 
+class Diagnostics;
 class Tokenizer;
 
 class DirectiveParser : public Lexer
 {
   public:
-    DirectiveParser(Tokenizer* tokenizer) : mTokenizer(tokenizer) { }
+    DirectiveParser(Tokenizer* tokenizer, Diagnostics* diagnostics);
 
     virtual void lex(Token* token);
 
@@ -41,6 +42,7 @@ class DirectiveParser : public Lexer
     void parseLine(Token* token);
 
     Tokenizer* mTokenizer;
+    Diagnostics* mDiagnostics;
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/ExpressionParser.cpp

@@ -93,12 +93,14 @@
 #include <cassert>
 #include <sstream>
 
+#include "Diagnostics.h"
 #include "Lexer.h"
 #include "Token.h"
 
 namespace {
 struct Context
 {
+    pp::Diagnostics* diagnostics;
     pp::Lexer* lexer;
     pp::Token* token;
     int* result;
@@ -454,9 +456,9 @@ static const yytype_int8 yyrhs[] =
 /* YYRLINE[YYN] -- source line where rule number YYN was defined.  */
 static const yytype_uint8 yyrline[] =
 {
-       0,    72,    72,    79,    80,    83,    86,    89,    92,    95,
-      98,   101,   104,   107,   110,   113,   116,   119,   122,   125,
-     136,   147,   150,   153,   156,   159,   162
+       0,    74,    74,    81,    82,    85,    88,    91,    94,    97,
+     100,   103,   106,   109,   112,   115,   118,   121,   124,   127,
+     140,   153,   156,   159,   162,   165,   168
 };
 #endif
 
@@ -1539,8 +1541,10 @@ yyreduce:
         if ((yyvsp[(3) - (3)]) == 0) {
             std::stringstream stream;
             stream << (yyvsp[(1) - (3)]) << " % " << (yyvsp[(3) - (3)]);
-            context->token->type = pp::Token::DIVISION_BY_ZERO;
-            context->token->value = stream.str();
+            std::string text = stream.str();
+            context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
+                                         context->token->location,
+                                         text.c_str());
             YYABORT;
         } else {
             (yyval) = (yyvsp[(1) - (3)]) % (yyvsp[(3) - (3)]);
@@ -1554,8 +1558,10 @@ yyreduce:
         if ((yyvsp[(3) - (3)]) == 0) {
             std::stringstream stream;
             stream << (yyvsp[(1) - (3)]) << " / " << (yyvsp[(3) - (3)]);
-            context->token->type = pp::Token::DIVISION_BY_ZERO;
-            context->token->value = stream.str();
+            std::string text = stream.str();
+            context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
+                                         context->token->location,
+                                         text.c_str());
             YYABORT;
         } else {
             (yyval) = (yyvsp[(1) - (3)]) / (yyvsp[(3) - (3)]);
@@ -1865,15 +1871,23 @@ int yylex(int* lvalp, Context* context)
 
 void yyerror(Context* context, const char* reason)
 {
-    context->token->type = pp::Token::INVALID_EXPRESSION;
-    context->token->value = reason;
+    context->diagnostics->report(pp::Diagnostics::INVALID_EXPRESSION,
+                                 context->token->location,
+                                 reason);
 }
 
 namespace pp {
 
+ExpressionParser::ExpressionParser(Lexer* lexer, Diagnostics* diagnostics) :
+    mLexer(lexer),
+    mDiagnostics(diagnostics)
+{
+}
+
 bool ExpressionParser::parse(Token* token, int* result)
 {
     Context context;
+    context.diagnostics = mDiagnostics;
     context.lexer = mLexer;
     context.token = token;
     context.result = result;
@@ -1885,14 +1899,12 @@ bool ExpressionParser::parse(Token* token, int* result)
         break;
 
       case 2:
-        token->type = pp::Token::OUT_OF_MEMORY;
-        token->value.clear();
+        mDiagnostics->report(Diagnostics::OUT_OF_MEMORY, token->location, "");
         break;
 
       default:
         assert(false);
-        token->type = pp::Token::INTERNAL_ERROR;
-        token->value.clear();
+        mDiagnostics->report(Diagnostics::INTERNAL_ERROR, token->location, "");
         break;
     }
 

src/compiler/preprocessor/new/ExpressionParser.h

@@ -12,13 +12,14 @@
 namespace pp
 {
 
+class Diagnostics;
 class Lexer;
 struct Token;
 
 class ExpressionParser
 {
   public:
-    ExpressionParser(Lexer* lexer) : mLexer(lexer) { }
+    ExpressionParser(Lexer* lexer, Diagnostics* diagnostics);
 
     bool parse(Token* token, int* result);
 
@@ -26,6 +27,7 @@ class ExpressionParser
     PP_DISALLOW_COPY_AND_ASSIGN(ExpressionParser);
 
     Lexer* mLexer;
+    Diagnostics* mDiagnostics;
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/ExpressionParser.y

@@ -30,12 +30,14 @@ WHICH GENERATES THE GLSL ES preprocessor expression parser.
 #include <cassert>
 #include <sstream>
 
+#include "Diagnostics.h"
 #include "Lexer.h"
 #include "Token.h"
 
 namespace {
 struct Context
 {
+    pp::Diagnostics* diagnostics;
     pp::Lexer* lexer;
     pp::Token* token;
     int* result;
@@ -126,8 +128,10 @@ expression
         if ($3 == 0) {
             std::stringstream stream;
             stream << $1 << " % " << $3;
-            context->token->type = pp::Token::DIVISION_BY_ZERO;
-            context->token->value = stream.str();
+            std::string text = stream.str();
+            context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
+                                         context->token->location,
+                                         text.c_str());
             YYABORT;
         } else {
             $$ = $1 % $3;
@@ -137,8 +141,10 @@ expression
         if ($3 == 0) {
             std::stringstream stream;
             stream << $1 << " / " << $3;
-            context->token->type = pp::Token::DIVISION_BY_ZERO;
-            context->token->value = stream.str();
+            std::string text = stream.str();
+            context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
+                                         context->token->location,
+                                         text.c_str());
             YYABORT;
         } else {
             $$ = $1 / $3;
@@ -212,15 +218,23 @@ int yylex(int* lvalp, Context* context)
 
 void yyerror(Context* context, const char* reason)
 {
-    context->token->type = pp::Token::INVALID_EXPRESSION;
-    context->token->value = reason;
+    context->diagnostics->report(pp::Diagnostics::INVALID_EXPRESSION,
+                                 context->token->location,
+                                 reason);
 }
 
 namespace pp {
 
+ExpressionParser::ExpressionParser(Lexer* lexer, Diagnostics* diagnostics) :
+    mLexer(lexer),
+    mDiagnostics(diagnostics)
+{
+}
+
 bool ExpressionParser::parse(Token* token, int* result)
 {
     Context context;
+    context.diagnostics = mDiagnostics;
     context.lexer = mLexer;
     context.token = token;
     context.result = result;
@@ -232,14 +246,12 @@ bool ExpressionParser::parse(Token* token, int* result)
         break;
 
       case 2:
-        token->type = pp::Token::OUT_OF_MEMORY;
-        token->value.clear();
+        mDiagnostics->report(Diagnostics::OUT_OF_MEMORY, token->location, "");
         break;
 
       default:
         assert(false);
-        token->type = pp::Token::INTERNAL_ERROR;
-        token->value.clear();
+        mDiagnostics->report(Diagnostics::INTERNAL_ERROR, token->location, "");
         break;
     }
 

src/compiler/preprocessor/new/MacroExpander.cpp

@@ -9,7 +9,9 @@
 namespace pp
 {
 
-MacroExpander::MacroExpander(Lexer* lexer) : mLexer(lexer)
+MacroExpander::MacroExpander(Lexer* lexer, Diagnostics* diagnostics) :
+    mLexer(lexer),
+    mDiagnostics(diagnostics)
 {
 }
 

src/compiler/preprocessor/new/MacroExpander.h

@@ -13,16 +13,20 @@
 namespace pp
 {
 
+class Diagnostics;
+
 class MacroExpander : public Lexer
 {
   public:
-    MacroExpander(Lexer* lexer);
+    MacroExpander(Lexer* lexer, Diagnostics* diagnostics);
 
     virtual void lex(Token* token);
 
   private:
     PP_DISALLOW_COPY_AND_ASSIGN(MacroExpander);
+
     Lexer* mLexer;
+    Diagnostics* mDiagnostics;
 };
 
 }  // namespace pp

src/compiler/preprocessor/new/Preprocessor.cpp

@@ -11,8 +11,11 @@
 namespace pp
 {
 
-Preprocessor::Preprocessor() : mDirectiveParser(&mTokenizer),
-                               mMacroExpander(&mDirectiveParser)
+Preprocessor::Preprocessor(Diagnostics* diagnostics) :
+    mDiagnostics(diagnostics),
+    mTokenizer(mDiagnostics),
+    mDirectiveParser(&mTokenizer, mDiagnostics),
+    mMacroExpander(&mDirectiveParser, mDiagnostics)
 {
 }
 
@@ -23,10 +26,9 @@ bool Preprocessor::init(int count,
     return mTokenizer.init(count, string, length);
 }
 
-int Preprocessor::lex(Token* token)
+void Preprocessor::lex(Token* token)
 {
     mMacroExpander.lex(token);
-    return token->type;
 }
 
 }  // namespace pp

src/compiler/preprocessor/new/Preprocessor.h

@@ -14,10 +14,12 @@
 namespace pp
 {
 
+class Diagnostics;
+
 class Preprocessor
 {
   public:
-    Preprocessor();
+    Preprocessor(Diagnostics* diagnostics);
 
     // count: specifies the number of elements in the string and length arrays.
     // string: specifies an array of pointers to strings.
@@ -30,11 +32,12 @@ class Preprocessor
     // is null terminated.
     bool init(int count, const char* const string[], const int length[]);
 
-    int lex(Token* token);
+    void lex(Token* token);
 
   private:
     PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
 
+    Diagnostics* mDiagnostics;
     Tokenizer mTokenizer;
     DirectiveParser mDirectiveParser;
     MacroExpander mMacroExpander;

src/compiler/preprocessor/new/SourceLocation.h

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef COMPILER_PREPROCESSOR_SOURCE_LOCATION_H_
+#define COMPILER_PREPROCESSOR_SOURCE_LOCATION_H_
+
+namespace pp
+{
+
+struct SourceLocation
+{
+    SourceLocation() : file(0), line(0) { }
+    SourceLocation(int f, int l) : file(f), line(l) { }
+
+    bool equals(const SourceLocation& other) const
+    {
+        return (file == other.file) && (line == other.line);
+    }
+
+    int file;
+    int line;
+};
+
+inline bool operator==(const SourceLocation& lhs, const SourceLocation& rhs)
+{
+    return lhs.equals(rhs);
+}
+
+inline bool operator!=(const SourceLocation& lhs, const SourceLocation& rhs)
+{
+    return !lhs.equals(rhs);
+}
+
+}  // namespace pp
+#endif  // COMPILER_PREPROCESSOR_SOURCE_LOCATION_H_

src/compiler/preprocessor/new/Token.h

@@ -10,6 +10,8 @@
 #include <ostream>
 #include <string>
 
+#include "SourceLocation.h"
+
 namespace pp
 {
 
@@ -17,20 +19,7 @@ struct Token
 {
     enum Type
     {
-        // Token IDs for error conditions are negative.
-        INTERNAL_ERROR = -1,
-        OUT_OF_MEMORY = -2,
-        INVALID_CHARACTER = -3,
-        INVALID_NUMBER = -4,
-        INVALID_DIRECTIVE = -5,
-        INVALID_EXPRESSION = -6,
-        DIVISION_BY_ZERO = -7,
-        EOF_IN_COMMENT = -8,
-        EOF_IN_DIRECTIVE = -9,
-        UNEXPECTED_TOKEN_IN_DIRECTIVE = -10,
-
-        // Indicates EOF.
-        LAST = 0,
+        LAST = 0,  // EOF.
 
         IDENTIFIER = 258,
 
@@ -63,17 +52,6 @@ struct Token
     {
         HAS_LEADING_SPACE = 1 << 0
     };
-    struct Location
-    {
-        Location() : file(0), line(0) { }
-        bool equals(const Location& other) const
-        {
-            return (file == other.file) && (line == other.line);
-        }
-
-        int file;
-        int line;
-    };
 
     Token() : type(0), flags(0) { }
 
@@ -81,7 +59,7 @@ struct Token
     {
         type = 0;
         flags = 0;
-        location = Location();
+        location = SourceLocation();
         value.clear();
     }
 
@@ -104,7 +82,7 @@ struct Token
 
     int type;
     int flags;
-    Location location;
+    SourceLocation location;
     std::string value;
 };
 

src/compiler/preprocessor/new/Tokenizer.cpp

@@ -515,10 +515,12 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 */
 
 #include "Tokenizer.h"
+
+#include "Diagnostics.h"
 #include "Token.h"
 
 typedef std::string YYSTYPE;
-typedef pp::Token::Location YYLTYPE;
+typedef pp::SourceLocation YYLTYPE;
 
 // Use the unused yycolumn variable to track file (string) number.
 #define yyfileno yycolumn
@@ -884,11 +886,18 @@ YY_RULE_SETUP
 { ++yylineno; }
 	YY_BREAK
 case YY_STATE_EOF(COMMENT):
-{ return pp::Token::EOF_IN_COMMENT; }
+{
+    yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+        pp::SourceLocation(yyfileno, yylineno), "");
+    yyterminate();
+}
 	YY_BREAK
 case 6:
 YY_RULE_SETUP
-{ yyextra->leadingSpace = true; BEGIN(INITIAL); }
+{
+    yyextra->leadingSpace = true;
+    BEGIN(INITIAL);
+}
 	YY_BREAK
 case 7:
 YY_RULE_SETUP
@@ -897,8 +906,9 @@ YY_RULE_SETUP
     if (yyextra->lineStart) {
         return yytext[0];
     } else {
-        yylval->assign(yytext, yyleng);
-        return pp::Token::INVALID_CHARACTER;
+        yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
+                                     pp::SourceLocation(yyfileno, yylineno),
+                                     std::string(yytext, yyleng));
     }
 }
 	YY_BREAK
@@ -928,8 +938,9 @@ YY_RULE_SETUP
 case 11:
 YY_RULE_SETUP
 {
-    yylval->assign(yytext, yyleng);
-    return pp::Token::INVALID_NUMBER;
+    yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
+                                 pp::SourceLocation(yyfileno, yylineno),
+                                 std::string(yytext, yyleng));
 }
 	YY_BREAK
 case 12:
@@ -1035,8 +1046,9 @@ YY_RULE_SETUP
 case 36:
 YY_RULE_SETUP
 {
-    yylval->assign(yytext, yyleng);
-    return pp::Token::INVALID_CHARACTER;
+    yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
+                                 pp::SourceLocation(yyfileno, yylineno),
+                                 std::string(yytext, yyleng));
 }
 	YY_BREAK
 case YY_STATE_EOF(INITIAL):
@@ -2183,8 +2195,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
 
 namespace pp {
 
-Tokenizer::Tokenizer() : mHandle(0)
+Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
+    mContext.diagnostics = diagnostics;
 }
 
 Tokenizer::~Tokenizer()

src/compiler/preprocessor/new/Tokenizer.h

@@ -14,11 +14,15 @@
 namespace pp
 {
 
+class Diagnostics;
+
 class Tokenizer : public Lexer
 {
   public:
     struct Context
     {
+        Diagnostics* diagnostics;
+
         Input input;
         // The location where yytext points to. Token location should track
         // scanLoc instead of Input::mReadLoc because they may not be the same
@@ -29,7 +33,7 @@ class Tokenizer : public Lexer
         bool lineStart;
     };
 
-    Tokenizer();
+    Tokenizer(Diagnostics* diagnostics);
     ~Tokenizer();
 
     bool init(int count, const char* const string[], const int length[]);

src/compiler/preprocessor/new/Tokenizer.l

@@ -24,10 +24,12 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 
 %{
 #include "Tokenizer.h"
+
+#include "Diagnostics.h"
 #include "Token.h"
 
 typedef std::string YYSTYPE;
-typedef pp::Token::Location YYLTYPE;
+typedef pp::SourceLocation YYLTYPE;
 
 // Use the unused yycolumn variable to track file (string) number.
 #define yyfileno yycolumn
@@ -89,16 +91,24 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 <COMMENT>[^*\r\n]+
 <COMMENT>"*"
 <COMMENT>{NEWLINE} { ++yylineno; }
-<COMMENT><<EOF>>    { return pp::Token::EOF_IN_COMMENT; }
-<COMMENT>"*/"       { yyextra->leadingSpace = true; BEGIN(INITIAL); }
+<COMMENT><<EOF>> {
+    yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+        pp::SourceLocation(yyfileno, yylineno), "");
+    yyterminate();
+}
+<COMMENT>"*/" {
+    yyextra->leadingSpace = true;
+    BEGIN(INITIAL);
+}
 
 # {
     // # is only valid at start of line for preprocessor directives.
     if (yyextra->lineStart) {
         return yytext[0];
     } else {
-        yylval->assign(yytext, yyleng);
-        return pp::Token::INVALID_CHARACTER;
+        yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
+                                     pp::SourceLocation(yyfileno, yylineno),
+                                     std::string(yytext, yyleng));
     }
 }
 
@@ -120,8 +130,9 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
     /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
     /* Rule to catch all invalid integers and floats. */
 ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
-    yylval->assign(yytext, yyleng);
-    return pp::Token::INVALID_NUMBER;
+    yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
+                                 pp::SourceLocation(yyfileno, yylineno),
+                                 std::string(yytext, yyleng));
 }
 
 "++"  { return pp::Token::OP_INC; }
@@ -155,8 +166,9 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 }
 
 . {
-    yylval->assign(yytext, yyleng);
-    return pp::Token::INVALID_CHARACTER;
+    yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
+                                 pp::SourceLocation(yyfileno, yylineno),
+                                 std::string(yytext, yyleng));
 }
 
 <<EOF>>   { yyterminate(); }
@@ -165,8 +177,9 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 namespace pp {
 
-Tokenizer::Tokenizer() : mHandle(0)
+Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
+    mContext.diagnostics = diagnostics;
 }
 
 Tokenizer::~Tokenizer()

tests/build_tests.gyp

@@ -47,6 +47,7 @@
         'preprocessor_tests/identifier_test.cpp',
         'preprocessor_tests/input_test.cpp',
         'preprocessor_tests/location_test.cpp',
+        'preprocessor_tests/MockDiagnostics.h',
         'preprocessor_tests/number_test.cpp',
         'preprocessor_tests/operator_test.cpp',
         'preprocessor_tests/token_test.cpp',

tests/preprocessor_tests/MockDiagnostics.h

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#ifndef PREPROCESSOR_TESTS_MOCK_DIAGNOSTICS_H_
+#define PREPROCESSOR_TESTS_MOCK_DIAGNOSTICS_H_
+
+#include "gmock/gmock.h"
+#include "Diagnostics.h"
+
+class MockDiagnostics : public pp::Diagnostics
+{
+  public:
+    MOCK_METHOD3(print,
+        void(ID id, const pp::SourceLocation& loc, const std::string& text));
+};
+
+#endif  // PREPROCESSOR_TESTS_MOCK_DIAGNOSTICS_H_

tests/preprocessor_tests/char_test.cpp

@@ -8,6 +8,8 @@
 #include <climits>
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
@@ -23,81 +25,75 @@ static const char kPunctuators[] = {
 static const int kNumPunctuators =
     sizeof(kPunctuators) / sizeof(kPunctuators[0]);
 
+bool isPunctuator(char c)
+{
+    static const char* kPunctuatorBeg = kPunctuators;
+    static const char* kPunctuatorEnd = kPunctuators + kNumPunctuators;
+    return std::find(kPunctuatorBeg, kPunctuatorEnd, c) != kPunctuatorEnd;
+}
+
 static const char kWhitespaces[] = {' ', '\t', '\v', '\f', '\n', '\r'};
 static const int kNumWhitespaces =
     sizeof(kWhitespaces) / sizeof(kWhitespaces[0]);
 
+bool isWhitespace(char c)
+{
+    static const char* kWhitespaceBeg = kWhitespaces;
+    static const char* kWhitespaceEnd = kWhitespaces + kNumWhitespaces;
+    return std::find(kWhitespaceBeg, kWhitespaceEnd, c) != kWhitespaceEnd;
+}
+
 TEST_P(CharTest, Identified)
 {
     std::string str(1, GetParam());
     const char* cstr = str.c_str();
     int length = 1;
 
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     // Note that we pass the length param as well because the invalid
     // string may contain the null character.
     ASSERT_TRUE(preprocessor.init(1, &cstr, &length));
 
-    pp::Token token;
-    int ret = preprocessor.lex(&token);
+    int expectedType = pp::Token::LAST;
+    std::string expectedValue;
 
-    // Handle identifier.
-    if ((cstr[0] == '_') ||
-        ((cstr[0] >= 'a') && (cstr[0] <= 'z')) ||
-        ((cstr[0] >= 'A') && (cstr[0] <= 'Z')))
+    if (str[0] == '#')
     {
-        EXPECT_EQ(pp::Token::IDENTIFIER, ret);
-        EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-        EXPECT_EQ(cstr[0], token.value[0]);
-        return;
+        // Lone '#' is ignored.
     }
-
-    // Handle numbers.
-    if (cstr[0] >= '0' && cstr[0] <= '9')
+    else if ((str[0] == '_') ||
+             ((str[0] >= 'a') && (str[0] <= 'z')) ||
+             ((str[0] >= 'A') && (str[0] <= 'Z')))
     {
-        EXPECT_EQ(pp::Token::CONST_INT, ret);
-        EXPECT_EQ(pp::Token::CONST_INT, token.type);
-        EXPECT_EQ(cstr[0], token.value[0]);
-        return;
+        expectedType = pp::Token::IDENTIFIER;
+        expectedValue = str;
     }
-
-    // Handle punctuators.
-    const char* lastIter = kPunctuators + kNumPunctuators;
-    const char* iter = std::find(kPunctuators, lastIter, cstr[0]);
-    if (iter != lastIter)
+    else if (str[0] >= '0' && str[0] <= '9')
     {
-        EXPECT_EQ(cstr[0], ret);
-        EXPECT_EQ(cstr[0], token.type);
-        EXPECT_TRUE(token.value.empty());
-        return;
+        expectedType = pp::Token::CONST_INT;
+        expectedValue = str;
     }
-
-    // Handle whitespace.
-    lastIter = kWhitespaces + kNumWhitespaces;
-    iter = std::find(kWhitespaces, lastIter, cstr[0]);
-    if (iter != lastIter)
+    else if (isPunctuator(str[0]))
+    {
+        expectedType = str[0];
+    }
+    else if (isWhitespace(str[0]))
     {
         // Whitespace is ignored.
-        EXPECT_EQ(pp::Token::LAST, ret);
-        EXPECT_EQ(pp::Token::LAST, token.type);
-        EXPECT_TRUE(token.value.empty());
-        return;
     }
-
-    // Handle number sign.
-    if (cstr[0] == '#')
+    else
     {
-        // Lone '#' is ignored.
-        EXPECT_EQ(pp::Token::LAST, ret);
-        EXPECT_EQ(pp::Token::LAST, token.type);
-        EXPECT_TRUE(token.value.empty());
-        return;
+        // Everything else is invalid.
+        using testing::_;
+        EXPECT_CALL(diagnostics,
+            print(pp::Diagnostics::INVALID_CHARACTER, _, str));
     }
 
-    // Everything else is invalid.
-    EXPECT_EQ(pp::Token::INVALID_CHARACTER, ret);
-    EXPECT_EQ(pp::Token::INVALID_CHARACTER, token.type);
-    EXPECT_EQ(cstr[0], token.value[0]);
+    pp::Token token;
+    preprocessor.lex(&token);
+    EXPECT_EQ(expectedType, token.type);
+    EXPECT_EQ(expectedValue, token.value);
 };
 
 // Note +1 for the max-value in range. It is there because the max-value

tests/preprocessor_tests/comment_test.cpp

@@ -5,6 +5,8 @@
 //
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
@@ -16,10 +18,12 @@ TEST_P(CommentTest, CommentIgnored)
 {
     const char* str = GetParam();
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::LAST, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::LAST, token.type);
 }
 
@@ -42,10 +46,12 @@ TEST(BlockComment, CommentReplacedWithSpace)
 {
     const char* str = "/*foo*/bar";
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ("bar", token.value.c_str());
     EXPECT_TRUE(token.hasLeadingSpace());
@@ -55,9 +61,13 @@ TEST(BlockComment, UnterminatedComment)
 {
     const char* str = "/*foo";
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::EOF_IN_COMMENT, preprocessor.lex(&token));
-    EXPECT_EQ(pp::Token::EOF_IN_COMMENT, token.type);
+
+    using testing::_;
+    EXPECT_CALL(diagnostics, print(pp::Diagnostics::EOF_IN_COMMENT, _, _));
+
+    pp::Token token;
+    preprocessor.lex(&token);
 }

tests/preprocessor_tests/identifier_test.cpp

@@ -5,15 +5,19 @@
 //
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
 static void PreprocessAndVerifyIdentifier(const char* str)
 {
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ(str, token.value.c_str());
 }

tests/preprocessor_tests/input_test.cpp

@@ -5,27 +5,33 @@
 //
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
 TEST(InputTest, NegativeCount)
 {
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     EXPECT_FALSE(preprocessor.init(-1, NULL, NULL));
 }
 
 TEST(InputTest, ZeroCount)
 {
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     EXPECT_TRUE(preprocessor.init(0, NULL, NULL));
-    EXPECT_EQ(pp::Token::LAST, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::LAST, token.type);
 }
 
 TEST(InputTest, NullString)
 {
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     EXPECT_FALSE(preprocessor.init(1, NULL, NULL));
 }
 

tests/preprocessor_tests/location_test.cpp

@@ -5,18 +5,21 @@
 //
 
 #include "gtest/gtest.h"
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
 static void PreprocessAndVerifyLocation(int count,
                                         const char* const string[],
                                         const int length[],
-                                        pp::Token::Location location)
+                                        const pp::SourceLocation& location)
 {
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(count, string, length));
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ("foo", token.value.c_str());
 
@@ -27,7 +30,7 @@ static void PreprocessAndVerifyLocation(int count,
 TEST(LocationTest, String0_Line1)
 {
     const char* str = "foo";
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 0;
     loc.line = 1;
 
@@ -38,7 +41,7 @@ TEST(LocationTest, String0_Line1)
 TEST(LocationTest, String0_Line2)
 {
     const char* str = "\nfoo";
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 0;
     loc.line = 2;
 
@@ -49,7 +52,7 @@ TEST(LocationTest, String0_Line2)
 TEST(LocationTest, String1_Line1)
 {
     const char* const str[] = {"\n\n", "foo"};
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 1;
     loc.line = 1;
 
@@ -60,7 +63,7 @@ TEST(LocationTest, String1_Line1)
 TEST(LocationTest, String1_Line2)
 {
     const char* const str[] = {"\n\n", "\nfoo"};
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 1;
     loc.line = 2;
 
@@ -71,7 +74,7 @@ TEST(LocationTest, String1_Line2)
 TEST(LocationTest, NewlineInsideCommentCounted)
 {
     const char* str = "/*\n\n*/foo";
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 0;
     loc.line = 3;
 
@@ -83,15 +86,16 @@ TEST(LocationTest, ErrorLocationAfterComment)
 {
     const char* str = "/*\n\n*/@";
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::INVALID_CHARACTER, preprocessor.lex(&token));
-    EXPECT_EQ(pp::Token::INVALID_CHARACTER, token.type);
-    EXPECT_STREQ("@", token.value.c_str());
 
-    EXPECT_EQ(0, token.location.file);
-    EXPECT_EQ(3, token.location.line);
+    pp::Diagnostics::ID id(pp::Diagnostics::INVALID_CHARACTER);
+    pp::SourceLocation loc(0, 3);
+    EXPECT_CALL(diagnostics, print(id, loc, "@"));
+
+    pp::Token token;
+    preprocessor.lex(&token);
 }
 
 // The location of a token straddling two or more strings is that of the
@@ -100,7 +104,7 @@ TEST(LocationTest, ErrorLocationAfterComment)
 TEST(LocationTest, TokenStraddlingTwoStrings)
 {
     const char* const str[] = {"f", "oo"};
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 0;
     loc.line = 1;
 
@@ -111,7 +115,7 @@ TEST(LocationTest, TokenStraddlingTwoStrings)
 TEST(LocationTest, TokenStraddlingThreeStrings)
 {
     const char* const str[] = {"f", "o", "o"};
-    pp::Token::Location loc;
+    pp::SourceLocation loc;
     loc.file = 0;
     loc.line = 1;
 

tests/preprocessor_tests/number_test.cpp

@@ -5,6 +5,8 @@
 //
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
@@ -18,12 +20,15 @@ TEST_P(InvalidNumberTest, InvalidNumberIdentified)
 {
     const char* str = GetParam();
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::INVALID_NUMBER, preprocessor.lex(&token));
-    EXPECT_EQ(pp::Token::INVALID_NUMBER, token.type);
-    EXPECT_STREQ(str, token.value.c_str());
+
+    using testing::_;
+    EXPECT_CALL(diagnostics, print(pp::Diagnostics::INVALID_NUMBER, _, str));
+
+    pp::Token token;
+    preprocessor.lex(&token);
 }
 
 INSTANTIATE_TEST_CASE_P(InvalidIntegers, InvalidNumberTest,
@@ -48,10 +53,12 @@ TEST_P(IntegerTest, IntegerIdentified)
     str.push_back(std::tr1::get<1>(GetParam()));  // digit.
     const char* cstr = str.c_str();
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &cstr, 0));
-    EXPECT_EQ(pp::Token::CONST_INT, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
     EXPECT_STREQ(cstr, token.value.c_str());
 }
@@ -85,10 +92,12 @@ INSTANTIATE_TEST_CASE_P(HexadecimalInteger_A_F,
 
 static void PreprocessAndVerifyFloat(const char* str)
 {
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
-    EXPECT_EQ(pp::Token::CONST_FLOAT, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::CONST_FLOAT, token.type);
     EXPECT_STREQ(str, token.value.c_str());
 }

tests/preprocessor_tests/operator_test.cpp

@@ -5,6 +5,8 @@
 //
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
@@ -24,10 +26,12 @@ TEST_P(OperatorTest, Identified)
 {
     OperatorTestParam param = GetParam();
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &param.str, 0));
-    EXPECT_EQ(param.op, preprocessor.lex(&token));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     EXPECT_EQ(param.op, token.type);
 }
 

tests/preprocessor_tests/space_test.cpp

@@ -5,6 +5,8 @@
 //
 
 #include "gtest/gtest.h"
+
+#include "MockDiagnostics.h"
 #include "Preprocessor.h"
 #include "Token.h"
 
@@ -29,11 +31,13 @@ TEST_P(SpaceCharTest, SpaceIgnored)
     str.append(identifier);
     const char* cstr = str.c_str();
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &cstr, 0));
+
+    pp::Token token;
     // Identifier "foo" is returned after ignoring the whitespace characters.
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ(identifier, token.value.c_str());
     // The whitespace character is however recorded with the next token.
@@ -66,11 +70,13 @@ TEST_P(SpaceStringTest, SpaceIgnored)
     str.append(identifier);
     const char* cstr = str.c_str();
 
-    pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &cstr, 0));
+
+    pp::Token token;
+    preprocessor.lex(&token);
     // Identifier "foo" is returned after ignoring the whitespace characters.
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ(identifier, token.value.c_str());
     // The whitespace character is however recorded with the next token.
@@ -93,23 +99,24 @@ TEST(SpaceTest, LeadingSpace)
     const char* str = " foo+ -bar";
 
     pp::Token token;
-    pp::Preprocessor preprocessor;
+    MockDiagnostics diagnostics;
+    pp::Preprocessor preprocessor(&diagnostics);
     ASSERT_TRUE(preprocessor.init(1, &str, 0));
 
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ("foo", token.value.c_str());
     EXPECT_TRUE(token.hasLeadingSpace());
 
-    EXPECT_EQ('+', preprocessor.lex(&token));
+    preprocessor.lex(&token);
     EXPECT_EQ('+', token.type);
     EXPECT_FALSE(token.hasLeadingSpace());
 
-    EXPECT_EQ('-', preprocessor.lex(&token));
+    preprocessor.lex(&token);
     EXPECT_EQ('-', token.type);
     EXPECT_TRUE(token.hasLeadingSpace());
 
-    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+    preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
     EXPECT_STREQ("bar", token.value.c_str());
     EXPECT_FALSE(token.hasLeadingSpace());