Skip to content

A
angle
Commit 2c958eef by alokp@chromium.org
Options

Moved error-handling to a separate class - Diagnostics. We were earlier…

Moved error-handling to a separate class - Diagnostics. We were earlier returning errors as tokens which did not work very well when error occured while parsing a preprocessor directive. Now all returned tokens are valid. Errors are reported via an abstract Diagnostics interface. Updated unit-tests with the new scheme. Review URL: https://codereview.appspot.com/6203089 git-svn-id: https://angleproject.googlecode.com/svn/trunk@1087 736b8ea6-26fd-11df-bfd4-992fa37f6226
parent 08365f68
Showing with 456 additions and 200 deletions
src/build_angle.gyp
......@@ -16,6 +16,8 @@
'include_dirs': [
],
'sources': [
'compiler/preprocessor/new/Diagnostics.cpp',
'compiler/preprocessor/new/Diagnostics.h',
'compiler/preprocessor/new/DirectiveParser.cpp',
'compiler/preprocessor/new/DirectiveParser.h',
'compiler/preprocessor/new/ExpressionParser.cpp',
......@@ -27,6 +29,7 @@
'compiler/preprocessor/new/MacroExpander.h',
'compiler/preprocessor/new/Preprocessor.cpp',
'compiler/preprocessor/new/Preprocessor.h',
'compiler/preprocessor/new/SourceLocation.h',
'compiler/preprocessor/new/Token.cpp',
'compiler/preprocessor/new/Token.h',
'compiler/preprocessor/new/Tokenizer.cpp',
......
src/compiler/preprocessor/new/Diagnostics.cpp 0 → 100644
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "Diagnostics.h"
#include <cassert>
namespace pp
{
void Diagnostics::report(ID id,
const SourceLocation& loc,
const std::string& text)
{
// TODO(alokp): Keep a count of errors and warnings.
print(id, loc, text);
}
Diagnostics::Severity Diagnostics::severity(ID id)
{
if ((id > ERROR_BEGIN) && (id < ERROR_END))
return ERROR;
if ((id > WARNING_BEGIN) && (id < WARNING_END))
return WARNING;
assert(false);
return ERROR;
}
} // namespace pp
src/compiler/preprocessor/new/Diagnostics.h 0 → 100644
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
#define COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
#include <string>
namespace pp
{
struct SourceLocation;
// Base class for reporting diagnostic messages.
// Derived classes are responsible for formatting and printing the messages.
class Diagnostics
{
public:
enum ID
{
ERROR_BEGIN,
INTERNAL_ERROR,
OUT_OF_MEMORY,
INVALID_CHARACTER,
INVALID_NUMBER,
INVALID_DIRECTIVE,
INVALID_EXPRESSION,
DIVISION_BY_ZERO,
EOF_IN_COMMENT,
EOF_IN_DIRECTIVE,
UNEXPECTED_TOKEN_IN_DIRECTIVE,
ERROR_END,
WARNING_BEGIN,
WARNING_END
};
void report(ID id, const SourceLocation& loc, const std::string& text);
protected:
enum Severity
{
ERROR,
WARNING
};
Severity severity(ID id);
virtual void print(ID id,
const SourceLocation& loc,
const std::string& text) = 0;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_DIAGNOSTICS_H_
src/compiler/preprocessor/new/DirectiveParser.cpp
......@@ -8,6 +8,7 @@
#include <cassert>
#include "Diagnostics.h"
#include "ExpressionParser.h"
#include "MacroExpander.h"
#include "Token.h"
......@@ -48,6 +49,13 @@ class DefinedParser : public Lexer
Lexer* mLexer;
};
DirectiveParser::DirectiveParser(Tokenizer* tokenizer,
Diagnostics* diagnostics) :
mTokenizer(tokenizer),
mDiagnostics(diagnostics)
{
}
void DirectiveParser::lex(Token* token)
{
do
......@@ -91,16 +99,24 @@ void DirectiveParser::parseDirective(Token* token)
else if (token->value == kDirectiveLine)
parseLine(token);
else
token->type = pp::Token::INVALID_DIRECTIVE;
mDiagnostics->report(Diagnostics::INVALID_DIRECTIVE,
token->location,
token->value.c_str());
}
if ((token->type != '\n') && (token->type != 0))
mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN_IN_DIRECTIVE,
token->location,
token->value.c_str());
while (token->type != '\n')
{
if (token->type == 0) {
//token->type = pp::Token::EOF_IN_DIRECTIVE;
mDiagnostics->report(Diagnostics::EOF_IN_DIRECTIVE,
token->location,
token->value.c_str());
break;
}
//token->type = pp::Token::INVALID_DIRECTIVE;
mTokenizer->lex(token);
}
}
......@@ -125,8 +141,8 @@ void DirectiveParser::parseIf(Token* token)
assert(token->value == kDirectiveIf);
DefinedParser definedParser(mTokenizer);
MacroExpander macroExpander(&definedParser);
ExpressionParser expressionParser(&macroExpander);
MacroExpander macroExpander(&definedParser, mDiagnostics);
ExpressionParser expressionParser(&macroExpander, mDiagnostics);
macroExpander.lex(token);
int expression = 0;
......@@ -207,7 +223,7 @@ void DirectiveParser::parseLine(Token* token)
{
// TODO(alokp): Implement me.
assert(token->value == kDirectiveLine);
MacroExpander macroExpander(mTokenizer);
MacroExpander macroExpander(mTokenizer, mDiagnostics);
macroExpander.lex(token);
}
......
src/compiler/preprocessor/new/DirectiveParser.h
......@@ -13,12 +13,13 @@
namespace pp
{
class Diagnostics;
class Tokenizer;
class DirectiveParser : public Lexer
{
public:
DirectiveParser(Tokenizer* tokenizer) : mTokenizer(tokenizer) { }
DirectiveParser(Tokenizer* tokenizer, Diagnostics* diagnostics);
virtual void lex(Token* token);
......@@ -41,6 +42,7 @@ class DirectiveParser : public Lexer
void parseLine(Token* token);
Tokenizer* mTokenizer;
Diagnostics* mDiagnostics;
};
} // namespace pp
......
src/compiler/preprocessor/new/ExpressionParser.cpp
......@@ -93,12 +93,14 @@
#include <cassert>
#include <sstream>
#include "Diagnostics.h"
#include "Lexer.h"
#include "Token.h"
namespace {
struct Context
{
pp::Diagnostics* diagnostics;
pp::Lexer* lexer;
pp::Token* token;
int* result;
......@@ -454,9 +456,9 @@ static const yytype_int8 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint8 yyrline[] =
{
0, 72, 72, 79, 80, 83, 86, 89, 92, 95,
98, 101, 104, 107, 110, 113, 116, 119, 122, 125,
136, 147, 150, 153, 156, 159, 162
0, 74, 74, 81, 82, 85, 88, 91, 94, 97,
100, 103, 106, 109, 112, 115, 118, 121, 124, 127,
140, 153, 156, 159, 162, 165, 168
};
#endif
......@@ -1539,8 +1541,10 @@ yyreduce:
if ((yyvsp[(3) - (3)]) == 0) {
std::stringstream stream;
stream << (yyvsp[(1) - (3)]) << " % " << (yyvsp[(3) - (3)]);
context->token->type = pp::Token::DIVISION_BY_ZERO;
context->token->value = stream.str();
std::string text = stream.str();
context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
context->token->location,
text.c_str());
YYABORT;
} else {
(yyval) = (yyvsp[(1) - (3)]) % (yyvsp[(3) - (3)]);
......@@ -1554,8 +1558,10 @@ yyreduce:
if ((yyvsp[(3) - (3)]) == 0) {
std::stringstream stream;
stream << (yyvsp[(1) - (3)]) << " / " << (yyvsp[(3) - (3)]);
context->token->type = pp::Token::DIVISION_BY_ZERO;
context->token->value = stream.str();
std::string text = stream.str();
context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
context->token->location,
text.c_str());
YYABORT;
} else {
(yyval) = (yyvsp[(1) - (3)]) / (yyvsp[(3) - (3)]);
......@@ -1865,15 +1871,23 @@ int yylex(int* lvalp, Context* context)
void yyerror(Context* context, const char* reason)
{
context->token->type = pp::Token::INVALID_EXPRESSION;
context->token->value = reason;
context->diagnostics->report(pp::Diagnostics::INVALID_EXPRESSION,
context->token->location,
reason);
}
namespace pp {
ExpressionParser::ExpressionParser(Lexer* lexer, Diagnostics* diagnostics) :
mLexer(lexer),
mDiagnostics(diagnostics)
{
}
bool ExpressionParser::parse(Token* token, int* result)
{
Context context;
context.diagnostics = mDiagnostics;
context.lexer = mLexer;
context.token = token;
context.result = result;
......@@ -1885,17 +1899,15 @@ bool ExpressionParser::parse(Token* token, int* result)
break;
case 2:
token->type = pp::Token::OUT_OF_MEMORY;
token->value.clear();
mDiagnostics->report(Diagnostics::OUT_OF_MEMORY, token->location, "");
break;
default:
assert(false);
token->type = pp::Token::INTERNAL_ERROR;
token->value.clear();
mDiagnostics->report(Diagnostics::INTERNAL_ERROR, token->location, "");
break;
}
return ret == 0;
}
......
src/compiler/preprocessor/new/ExpressionParser.h
......@@ -12,13 +12,14 @@
namespace pp
{
class Diagnostics;
class Lexer;
struct Token;
class ExpressionParser
{
public:
ExpressionParser(Lexer* lexer) : mLexer(lexer) { }
ExpressionParser(Lexer* lexer, Diagnostics* diagnostics);
bool parse(Token* token, int* result);
......@@ -26,6 +27,7 @@ class ExpressionParser
PP_DISALLOW_COPY_AND_ASSIGN(ExpressionParser);
Lexer* mLexer;
Diagnostics* mDiagnostics;
};
} // namespace pp
......
src/compiler/preprocessor/new/ExpressionParser.y
......@@ -30,12 +30,14 @@ WHICH GENERATES THE GLSL ES preprocessor expression parser.
#include <cassert>
#include <sstream>
#include "Diagnostics.h"
#include "Lexer.h"
#include "Token.h"
namespace {
struct Context
{
pp::Diagnostics* diagnostics;
pp::Lexer* lexer;
pp::Token* token;
int* result;
......@@ -126,8 +128,10 @@ expression
if ($3 == 0) {
std::stringstream stream;
stream << $1 << " % " << $3;
context->token->type = pp::Token::DIVISION_BY_ZERO;
context->token->value = stream.str();
std::string text = stream.str();
context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
context->token->location,
text.c_str());
YYABORT;
} else {
$$ = $1 % $3;
......@@ -137,8 +141,10 @@ expression
if ($3 == 0) {
std::stringstream stream;
stream << $1 << " / " << $3;
context->token->type = pp::Token::DIVISION_BY_ZERO;
context->token->value = stream.str();
std::string text = stream.str();
context->diagnostics->report(pp::Diagnostics::DIVISION_BY_ZERO,
context->token->location,
text.c_str());
YYABORT;
} else {
$$ = $1 / $3;
......@@ -212,15 +218,23 @@ int yylex(int* lvalp, Context* context)
void yyerror(Context* context, const char* reason)
{
context->token->type = pp::Token::INVALID_EXPRESSION;
context->token->value = reason;
context->diagnostics->report(pp::Diagnostics::INVALID_EXPRESSION,
context->token->location,
reason);
}
namespace pp {
ExpressionParser::ExpressionParser(Lexer* lexer, Diagnostics* diagnostics) :
mLexer(lexer),
mDiagnostics(diagnostics)
{
}
bool ExpressionParser::parse(Token* token, int* result)
{
Context context;
context.diagnostics = mDiagnostics;
context.lexer = mLexer;
context.token = token;
context.result = result;
......@@ -232,17 +246,15 @@ bool ExpressionParser::parse(Token* token, int* result)
break;
case 2:
token->type = pp::Token::OUT_OF_MEMORY;
token->value.clear();
mDiagnostics->report(Diagnostics::OUT_OF_MEMORY, token->location, "");
break;
default:
assert(false);
token->type = pp::Token::INTERNAL_ERROR;
token->value.clear();
mDiagnostics->report(Diagnostics::INTERNAL_ERROR, token->location, "");
break;
}
return ret == 0;
}
......
src/compiler/preprocessor/new/MacroExpander.cpp
......@@ -9,7 +9,9 @@
namespace pp
{
MacroExpander::MacroExpander(Lexer* lexer) : mLexer(lexer)
MacroExpander::MacroExpander(Lexer* lexer, Diagnostics* diagnostics) :
mLexer(lexer),
mDiagnostics(diagnostics)
{
}
......
src/compiler/preprocessor/new/MacroExpander.h
......@@ -13,16 +13,20 @@
namespace pp
{
class Diagnostics;
class MacroExpander : public Lexer
{
public:
MacroExpander(Lexer* lexer);
MacroExpander(Lexer* lexer, Diagnostics* diagnostics);
virtual void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(MacroExpander);
Lexer* mLexer;
Diagnostics* mDiagnostics;
};
} // namespace pp
......
src/compiler/preprocessor/new/Preprocessor.cpp
......@@ -11,8 +11,11 @@
namespace pp
{
Preprocessor::Preprocessor() : mDirectiveParser(&mTokenizer),
mMacroExpander(&mDirectiveParser)
Preprocessor::Preprocessor(Diagnostics* diagnostics) :
mDiagnostics(diagnostics),
mTokenizer(mDiagnostics),
mDirectiveParser(&mTokenizer, mDiagnostics),
mMacroExpander(&mDirectiveParser, mDiagnostics)
{
}
......@@ -23,10 +26,9 @@ bool Preprocessor::init(int count,
return mTokenizer.init(count, string, length);
}
int Preprocessor::lex(Token* token)
void Preprocessor::lex(Token* token)
{
mMacroExpander.lex(token);
return token->type;
}
} // namespace pp
......
src/compiler/preprocessor/new/Preprocessor.h
......@@ -14,10 +14,12 @@
namespace pp
{
class Diagnostics;
class Preprocessor
{
public:
Preprocessor();
Preprocessor(Diagnostics* diagnostics);
// count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings.
......@@ -30,11 +32,12 @@ class Preprocessor
// is null terminated.
bool init(int count, const char* const string[], const int length[]);
int lex(Token* token);
void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
Diagnostics* mDiagnostics;
Tokenizer mTokenizer;
DirectiveParser mDirectiveParser;
MacroExpander mMacroExpander;
......
src/compiler/preprocessor/new/SourceLocation.h 0 → 100644
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_SOURCE_LOCATION_H_
#define COMPILER_PREPROCESSOR_SOURCE_LOCATION_H_
namespace pp
{
struct SourceLocation
{
SourceLocation() : file(0), line(0) { }
SourceLocation(int f, int l) : file(f), line(l) { }
bool equals(const SourceLocation& other) const
{
return (file == other.file) && (line == other.line);
}
int file;
int line;
};
inline bool operator==(const SourceLocation& lhs, const SourceLocation& rhs)
{
return lhs.equals(rhs);
}
inline bool operator!=(const SourceLocation& lhs, const SourceLocation& rhs)
{
return !lhs.equals(rhs);
}
} // namespace pp
#endif // COMPILER_PREPROCESSOR_SOURCE_LOCATION_H_
src/compiler/preprocessor/new/Token.h
......@@ -10,6 +10,8 @@
#include <ostream>
#include <string>
#include "SourceLocation.h"
namespace pp
{
......@@ -17,20 +19,7 @@ struct Token
{
enum Type
{
// Token IDs for error conditions are negative.
INTERNAL_ERROR = -1,
OUT_OF_MEMORY = -2,
INVALID_CHARACTER = -3,
INVALID_NUMBER = -4,
INVALID_DIRECTIVE = -5,
INVALID_EXPRESSION = -6,
DIVISION_BY_ZERO = -7,
EOF_IN_COMMENT = -8,
EOF_IN_DIRECTIVE = -9,
UNEXPECTED_TOKEN_IN_DIRECTIVE = -10,
// Indicates EOF.
LAST = 0,
LAST = 0, // EOF.
IDENTIFIER = 258,
......@@ -63,17 +52,6 @@ struct Token
{
HAS_LEADING_SPACE = 1 << 0
};
struct Location
{
Location() : file(0), line(0) { }
bool equals(const Location& other) const
{
return (file == other.file) && (line == other.line);
}
int file;
int line;
};
Token() : type(0), flags(0) { }
......@@ -81,7 +59,7 @@ struct Token
{
type = 0;
flags = 0;
location = Location();
location = SourceLocation();
value.clear();
}
......@@ -104,7 +82,7 @@ struct Token
int type;
int flags;
Location location;
SourceLocation location;
std::string value;
};
......
src/compiler/preprocessor/new/Tokenizer.cpp
......@@ -515,10 +515,12 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/
#include "Tokenizer.h"
#include "Diagnostics.h"
#include "Token.h"
typedef std::string YYSTYPE;
typedef pp::Token::Location YYLTYPE;
typedef pp::SourceLocation YYLTYPE;
// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn
......@@ -884,11 +886,18 @@ YY_RULE_SETUP
{ ++yylineno; }
YY_BREAK
case YY_STATE_EOF(COMMENT):
{ return pp::Token::EOF_IN_COMMENT; }
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno), "");
yyterminate();
}
YY_BREAK
case 6:
YY_RULE_SETUP
{ yyextra->leadingSpace = true; BEGIN(INITIAL); }
{
yyextra->leadingSpace = true;
BEGIN(INITIAL);
}
YY_BREAK
case 7:
YY_RULE_SETUP
......@@ -897,8 +906,9 @@ YY_RULE_SETUP
if (yyextra->lineStart) {
return yytext[0];
} else {
yylval->assign(yytext, yyleng);
return pp::Token::INVALID_CHARACTER;
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
pp::SourceLocation(yyfileno, yylineno),
std::string(yytext, yyleng));
}
}
YY_BREAK
......@@ -928,8 +938,9 @@ YY_RULE_SETUP
case 11:
YY_RULE_SETUP
{
yylval->assign(yytext, yyleng);
return pp::Token::INVALID_NUMBER;
yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
pp::SourceLocation(yyfileno, yylineno),
std::string(yytext, yyleng));
}
YY_BREAK
case 12:
......@@ -1035,8 +1046,9 @@ YY_RULE_SETUP
case 36:
YY_RULE_SETUP
{
yylval->assign(yytext, yyleng);
return pp::Token::INVALID_CHARACTER;
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
pp::SourceLocation(yyfileno, yylineno),
std::string(yytext, yyleng));
}
YY_BREAK
case YY_STATE_EOF(INITIAL):
......@@ -2183,8 +2195,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
namespace pp {
Tokenizer::Tokenizer() : mHandle(0)
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
{
mContext.diagnostics = diagnostics;
}
Tokenizer::~Tokenizer()
......
src/compiler/preprocessor/new/Tokenizer.h
......@@ -14,11 +14,15 @@
namespace pp
{
class Diagnostics;
class Tokenizer : public Lexer
{
public:
struct Context
{
Diagnostics* diagnostics;
Input input;
// The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same
......@@ -29,7 +33,7 @@ class Tokenizer : public Lexer
bool lineStart;
};
Tokenizer();
Tokenizer(Diagnostics* diagnostics);
~Tokenizer();
bool init(int count, const char* const string[], const int length[]);
......
src/compiler/preprocessor/new/Tokenizer.l
......@@ -24,10 +24,12 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
%{
#include "Tokenizer.h"
#include "Diagnostics.h"
#include "Token.h"
typedef std::string YYSTYPE;
typedef pp::Token::Location YYLTYPE;
typedef pp::SourceLocation YYLTYPE;
// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn
......@@ -85,20 +87,28 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
/* Block comment */
/* Line breaks are just counted - not returned. */
/* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT><<EOF>> { return pp::Token::EOF_IN_COMMENT; }
<COMMENT>"*/" { yyextra->leadingSpace = true; BEGIN(INITIAL); }
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT><<EOF>> {
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno), "");
yyterminate();
}
<COMMENT>"*/" {
yyextra->leadingSpace = true;
BEGIN(INITIAL);
}
# {
// # is only valid at start of line for preprocessor directives.
if (yyextra->lineStart) {
return yytext[0];
} else {
yylval->assign(yytext, yyleng);
return pp::Token::INVALID_CHARACTER;
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
pp::SourceLocation(yyfileno, yylineno),
std::string(yytext, yyleng));
}
}
......@@ -120,8 +130,9 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
/* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
/* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
yylval->assign(yytext, yyleng);
return pp::Token::INVALID_NUMBER;
yyextra->diagnostics->report(pp::Diagnostics::INVALID_NUMBER,
pp::SourceLocation(yyfileno, yylineno),
std::string(yytext, yyleng));
}
"++" { return pp::Token::OP_INC; }
......@@ -155,8 +166,9 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
}
. {
yylval->assign(yytext, yyleng);
return pp::Token::INVALID_CHARACTER;
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
pp::SourceLocation(yyfileno, yylineno),
std::string(yytext, yyleng));
}
<<EOF>> { yyterminate(); }
......@@ -165,8 +177,9 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
namespace pp {
Tokenizer::Tokenizer() : mHandle(0)
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
{
mContext.diagnostics = diagnostics;
}
Tokenizer::~Tokenizer()
......
tests/build_tests.gyp
......@@ -47,6 +47,7 @@
'preprocessor_tests/identifier_test.cpp',
'preprocessor_tests/input_test.cpp',
'preprocessor_tests/location_test.cpp',
'preprocessor_tests/MockDiagnostics.h',
'preprocessor_tests/number_test.cpp',
'preprocessor_tests/operator_test.cpp',
'preprocessor_tests/token_test.cpp',
......
tests/preprocessor_tests/MockDiagnostics.h 0 → 100644
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef PREPROCESSOR_TESTS_MOCK_DIAGNOSTICS_H_
#define PREPROCESSOR_TESTS_MOCK_DIAGNOSTICS_H_
#include "gmock/gmock.h"
#include "Diagnostics.h"
class MockDiagnostics : public pp::Diagnostics
{
public:
MOCK_METHOD3(print,
void(ID id, const pp::SourceLocation& loc, const std::string& text));
};
#endif // PREPROCESSOR_TESTS_MOCK_DIAGNOSTICS_H_
tests/preprocessor_tests/char_test.cpp
......@@ -8,6 +8,8 @@
#include <climits>
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
......@@ -23,81 +25,75 @@ static const char kPunctuators[] = {
static const int kNumPunctuators =
sizeof(kPunctuators) / sizeof(kPunctuators[0]);
bool isPunctuator(char c)
{
static const char* kPunctuatorBeg = kPunctuators;
static const char* kPunctuatorEnd = kPunctuators + kNumPunctuators;
return std::find(kPunctuatorBeg, kPunctuatorEnd, c) != kPunctuatorEnd;
}
static const char kWhitespaces[] = {' ', '\t', '\v', '\f', '\n', '\r'};
static const int kNumWhitespaces =
sizeof(kWhitespaces) / sizeof(kWhitespaces[0]);
bool isWhitespace(char c)
{
static const char* kWhitespaceBeg = kWhitespaces;
static const char* kWhitespaceEnd = kWhitespaces + kNumWhitespaces;
return std::find(kWhitespaceBeg, kWhitespaceEnd, c) != kWhitespaceEnd;
}
TEST_P(CharTest, Identified)
{
std::string str(1, GetParam());
const char* cstr = str.c_str();
int length = 1;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
// Note that we pass the length param as well because the invalid
// string may contain the null character.
ASSERT_TRUE(preprocessor.init(1, &cstr, &length));
pp::Token token;
int ret = preprocessor.lex(&token);
int expectedType = pp::Token::LAST;
std::string expectedValue;
// Handle identifier.
if ((cstr[0] == '_') ||
((cstr[0] >= 'a') && (cstr[0] <= 'z')) ||
((cstr[0] >= 'A') && (cstr[0] <= 'Z')))
if (str[0] == '#')
{
EXPECT_EQ(pp::Token::IDENTIFIER, ret);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_EQ(cstr[0], token.value[0]);
return;
// Lone '#' is ignored.
}
// Handle numbers.
if (cstr[0] >= '0' && cstr[0] <= '9')
else if ((str[0] == '_') ||
((str[0] >= 'a') && (str[0] <= 'z')) ||
((str[0] >= 'A') && (str[0] <= 'Z')))
{
EXPECT_EQ(pp::Token::CONST_INT, ret);
EXPECT_EQ(pp::Token::CONST_INT, token.type);
EXPECT_EQ(cstr[0], token.value[0]);
return;
expectedType = pp::Token::IDENTIFIER;
expectedValue = str;
}
// Handle punctuators.
const char* lastIter = kPunctuators + kNumPunctuators;
const char* iter = std::find(kPunctuators, lastIter, cstr[0]);
if (iter != lastIter)
else if (str[0] >= '0' && str[0] <= '9')
{
EXPECT_EQ(cstr[0], ret);
EXPECT_EQ(cstr[0], token.type);
EXPECT_TRUE(token.value.empty());
return;
expectedType = pp::Token::CONST_INT;
expectedValue = str;
}
// Handle whitespace.
lastIter = kWhitespaces + kNumWhitespaces;
iter = std::find(kWhitespaces, lastIter, cstr[0]);
if (iter != lastIter)
else if (isPunctuator(str[0]))
{
expectedType = str[0];
}
else if (isWhitespace(str[0]))
{
// Whitespace is ignored.
EXPECT_EQ(pp::Token::LAST, ret);
EXPECT_EQ(pp::Token::LAST, token.type);
EXPECT_TRUE(token.value.empty());
return;
}
// Handle number sign.
if (cstr[0] == '#')
else
{
// Lone '#' is ignored.
EXPECT_EQ(pp::Token::LAST, ret);
EXPECT_EQ(pp::Token::LAST, token.type);
EXPECT_TRUE(token.value.empty());
return;
// Everything else is invalid.
using testing::_;
EXPECT_CALL(diagnostics,
print(pp::Diagnostics::INVALID_CHARACTER, _, str));
}
// Everything else is invalid.
EXPECT_EQ(pp::Token::INVALID_CHARACTER, ret);
EXPECT_EQ(pp::Token::INVALID_CHARACTER, token.type);
EXPECT_EQ(cstr[0], token.value[0]);
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(expectedType, token.type);
EXPECT_EQ(expectedValue, token.value);
};
// Note +1 for the max-value in range. It is there because the max-value
......
tests/preprocessor_tests/comment_test.cpp
......@@ -5,6 +5,8 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
......@@ -16,10 +18,12 @@ TEST_P(CommentTest, CommentIgnored)
{
const char* str = GetParam();
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::LAST, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::LAST, token.type);
}
......@@ -42,10 +46,12 @@ TEST(BlockComment, CommentReplacedWithSpace)
{
const char* str = "/*foo*/bar";
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("bar", token.value.c_str());
EXPECT_TRUE(token.hasLeadingSpace());
......@@ -55,9 +61,13 @@ TEST(BlockComment, UnterminatedComment)
{
const char* str = "/*foo";
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::EOF_IN_COMMENT, preprocessor.lex(&token));
EXPECT_EQ(pp::Token::EOF_IN_COMMENT, token.type);
using testing::_;
EXPECT_CALL(diagnostics, print(pp::Diagnostics::EOF_IN_COMMENT, _, _));
pp::Token token;
preprocessor.lex(&token);
}
tests/preprocessor_tests/identifier_test.cpp
......@@ -5,15 +5,19 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
static void PreprocessAndVerifyIdentifier(const char* str)
{
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ(str, token.value.c_str());
}
......
tests/preprocessor_tests/input_test.cpp
......@@ -5,27 +5,33 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
TEST(InputTest, NegativeCount)
{
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
EXPECT_FALSE(preprocessor.init(-1, NULL, NULL));
}
TEST(InputTest, ZeroCount)
{
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
EXPECT_TRUE(preprocessor.init(0, NULL, NULL));
EXPECT_EQ(pp::Token::LAST, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::LAST, token.type);
}
TEST(InputTest, NullString)
{
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
EXPECT_FALSE(preprocessor.init(1, NULL, NULL));
}
......
tests/preprocessor_tests/location_test.cpp
......@@ -5,18 +5,21 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
static void PreprocessAndVerifyLocation(int count,
const char* const string[],
const int length[],
pp::Token::Location location)
const pp::SourceLocation& location)
{
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(count, string, length));
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("foo", token.value.c_str());
......@@ -27,7 +30,7 @@ static void PreprocessAndVerifyLocation(int count,
TEST(LocationTest, String0_Line1)
{
const char* str = "foo";
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 0;
loc.line = 1;
......@@ -38,7 +41,7 @@ TEST(LocationTest, String0_Line1)
TEST(LocationTest, String0_Line2)
{
const char* str = "\nfoo";
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 0;
loc.line = 2;
......@@ -49,7 +52,7 @@ TEST(LocationTest, String0_Line2)
TEST(LocationTest, String1_Line1)
{
const char* const str[] = {"\n\n", "foo"};
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 1;
loc.line = 1;
......@@ -60,7 +63,7 @@ TEST(LocationTest, String1_Line1)
TEST(LocationTest, String1_Line2)
{
const char* const str[] = {"\n\n", "\nfoo"};
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 1;
loc.line = 2;
......@@ -71,7 +74,7 @@ TEST(LocationTest, String1_Line2)
TEST(LocationTest, NewlineInsideCommentCounted)
{
const char* str = "/*\n\n*/foo";
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 0;
loc.line = 3;
......@@ -83,15 +86,16 @@ TEST(LocationTest, ErrorLocationAfterComment)
{
const char* str = "/*\n\n*/@";
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::INVALID_CHARACTER, preprocessor.lex(&token));
EXPECT_EQ(pp::Token::INVALID_CHARACTER, token.type);
EXPECT_STREQ("@", token.value.c_str());
EXPECT_EQ(0, token.location.file);
EXPECT_EQ(3, token.location.line);
pp::Diagnostics::ID id(pp::Diagnostics::INVALID_CHARACTER);
pp::SourceLocation loc(0, 3);
EXPECT_CALL(diagnostics, print(id, loc, "@"));
pp::Token token;
preprocessor.lex(&token);
}
// The location of a token straddling two or more strings is that of the
......@@ -100,7 +104,7 @@ TEST(LocationTest, ErrorLocationAfterComment)
TEST(LocationTest, TokenStraddlingTwoStrings)
{
const char* const str[] = {"f", "oo"};
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 0;
loc.line = 1;
......@@ -111,7 +115,7 @@ TEST(LocationTest, TokenStraddlingTwoStrings)
TEST(LocationTest, TokenStraddlingThreeStrings)
{
const char* const str[] = {"f", "o", "o"};
pp::Token::Location loc;
pp::SourceLocation loc;
loc.file = 0;
loc.line = 1;
......
tests/preprocessor_tests/number_test.cpp
......@@ -5,6 +5,8 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
......@@ -18,12 +20,15 @@ TEST_P(InvalidNumberTest, InvalidNumberIdentified)
{
const char* str = GetParam();
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::INVALID_NUMBER, preprocessor.lex(&token));
EXPECT_EQ(pp::Token::INVALID_NUMBER, token.type);
EXPECT_STREQ(str, token.value.c_str());
using testing::_;
EXPECT_CALL(diagnostics, print(pp::Diagnostics::INVALID_NUMBER, _, str));
pp::Token token;
preprocessor.lex(&token);
}
INSTANTIATE_TEST_CASE_P(InvalidIntegers, InvalidNumberTest,
......@@ -48,10 +53,12 @@ TEST_P(IntegerTest, IntegerIdentified)
str.push_back(std::tr1::get<1>(GetParam())); // digit.
const char* cstr = str.c_str();
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &cstr, 0));
EXPECT_EQ(pp::Token::CONST_INT, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::CONST_INT, token.type);
EXPECT_STREQ(cstr, token.value.c_str());
}
......@@ -85,10 +92,12 @@ INSTANTIATE_TEST_CASE_P(HexadecimalInteger_A_F,
static void PreprocessAndVerifyFloat(const char* str)
{
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::CONST_FLOAT, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::CONST_FLOAT, token.type);
EXPECT_STREQ(str, token.value.c_str());
}
......
tests/preprocessor_tests/operator_test.cpp
......@@ -5,6 +5,8 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
......@@ -24,10 +26,12 @@ TEST_P(OperatorTest, Identified)
{
OperatorTestParam param = GetParam();
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &param.str, 0));
EXPECT_EQ(param.op, preprocessor.lex(&token));
pp::Token token;
preprocessor.lex(&token);
EXPECT_EQ(param.op, token.type);
}
......
tests/preprocessor_tests/space_test.cpp
......@@ -5,6 +5,8 @@
//
#include "gtest/gtest.h"
#include "MockDiagnostics.h"
#include "Preprocessor.h"
#include "Token.h"
......@@ -29,11 +31,13 @@ TEST_P(SpaceCharTest, SpaceIgnored)
str.append(identifier);
const char* cstr = str.c_str();
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &cstr, 0));
pp::Token token;
// Identifier "foo" is returned after ignoring the whitespace characters.
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ(identifier, token.value.c_str());
// The whitespace character is however recorded with the next token.
......@@ -66,11 +70,13 @@ TEST_P(SpaceStringTest, SpaceIgnored)
str.append(identifier);
const char* cstr = str.c_str();
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &cstr, 0));
pp::Token token;
preprocessor.lex(&token);
// Identifier "foo" is returned after ignoring the whitespace characters.
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ(identifier, token.value.c_str());
// The whitespace character is however recorded with the next token.
......@@ -93,23 +99,24 @@ TEST(SpaceTest, LeadingSpace)
const char* str = " foo+ -bar";
pp::Token token;
pp::Preprocessor preprocessor;
MockDiagnostics diagnostics;
pp::Preprocessor preprocessor(&diagnostics);
ASSERT_TRUE(preprocessor.init(1, &str, 0));
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("foo", token.value.c_str());
EXPECT_TRUE(token.hasLeadingSpace());
EXPECT_EQ('+', preprocessor.lex(&token));
preprocessor.lex(&token);
EXPECT_EQ('+', token.type);
EXPECT_FALSE(token.hasLeadingSpace());
EXPECT_EQ('-', preprocessor.lex(&token));
preprocessor.lex(&token);
EXPECT_EQ('-', token.type);
EXPECT_TRUE(token.hasLeadingSpace());
EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("bar", token.value.c_str());
EXPECT_FALSE(token.hasLeadingSpace());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment