Commit 07d921de by alokp@chromium.org

Token::value now contains text for all types of tokens - not just CONST_INT,…

Token::value now contains text for all types of tokens - not just CONST_INT, CONST_FLOAT, and IDENTIFIER. This makes it easier to report diagnostics and various preprocessor directives. git-svn-id: https://angleproject.googlecode.com/svn/trunk@1093 736b8ea6-26fd-11df-bfd4-992fa37f6226
parent 98d04ec8
...@@ -885,13 +885,6 @@ case 5: ...@@ -885,13 +885,6 @@ case 5:
YY_RULE_SETUP YY_RULE_SETUP
{ ++yylineno; } { ++yylineno; }
YY_BREAK YY_BREAK
case YY_STATE_EOF(COMMENT):
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno), "");
yyterminate();
}
YY_BREAK
case 6: case 6:
YY_RULE_SETUP YY_RULE_SETUP
{ {
...@@ -904,6 +897,7 @@ YY_RULE_SETUP ...@@ -904,6 +897,7 @@ YY_RULE_SETUP
{ {
// # is only valid at start of line for preprocessor directives. // # is only valid at start of line for preprocessor directives.
if (yyextra->lineStart) { if (yyextra->lineStart) {
yylval->assign(1, yytext[0]);
return yytext[0]; return yytext[0];
} else { } else {
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER, yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
...@@ -945,91 +939,157 @@ YY_RULE_SETUP ...@@ -945,91 +939,157 @@ YY_RULE_SETUP
YY_BREAK YY_BREAK
case 12: case 12:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_INC; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_INC;
}
YY_BREAK YY_BREAK
case 13: case 13:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_DEC; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DEC;
}
YY_BREAK YY_BREAK
case 14: case 14:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_LEFT; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT;
}
YY_BREAK YY_BREAK
case 15: case 15:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_RIGHT; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT;
}
YY_BREAK YY_BREAK
case 16: case 16:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_LE; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LE;
}
YY_BREAK YY_BREAK
case 17: case 17:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_GE; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_GE;
}
YY_BREAK YY_BREAK
case 18: case 18:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_EQ; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_EQ;
}
YY_BREAK YY_BREAK
case 19: case 19:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_NE; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_NE;
}
YY_BREAK YY_BREAK
case 20: case 20:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_AND; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND;
}
YY_BREAK YY_BREAK
case 21: case 21:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_XOR; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR;
}
YY_BREAK YY_BREAK
case 22: case 22:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_OR; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR;
}
YY_BREAK YY_BREAK
case 23: case 23:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_ADD_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_ADD_ASSIGN;
}
YY_BREAK YY_BREAK
case 24: case 24:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_SUB_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_SUB_ASSIGN;
}
YY_BREAK YY_BREAK
case 25: case 25:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_MUL_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MUL_ASSIGN;
}
YY_BREAK YY_BREAK
case 26: case 26:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_DIV_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DIV_ASSIGN;
}
YY_BREAK YY_BREAK
case 27: case 27:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_MOD_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MOD_ASSIGN;
}
YY_BREAK YY_BREAK
case 28: case 28:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_LEFT_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT_ASSIGN;
}
YY_BREAK YY_BREAK
case 29: case 29:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_RIGHT_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT_ASSIGN;
}
YY_BREAK YY_BREAK
case 30: case 30:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_AND_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND_ASSIGN;
}
YY_BREAK YY_BREAK
case 31: case 31:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_XOR_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR_ASSIGN;
}
YY_BREAK YY_BREAK
case 32: case 32:
YY_RULE_SETUP YY_RULE_SETUP
{ return pp::Token::OP_OR_ASSIGN; } {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR_ASSIGN;
}
YY_BREAK YY_BREAK
case 33: case 33:
YY_RULE_SETUP YY_RULE_SETUP
{ return yytext[0]; } {
yylval->assign(1, yytext[0]);
return yytext[0];
}
YY_BREAK YY_BREAK
case 34: case 34:
YY_RULE_SETUP YY_RULE_SETUP
...@@ -1040,6 +1100,7 @@ case 35: ...@@ -1040,6 +1100,7 @@ case 35:
YY_RULE_SETUP YY_RULE_SETUP
{ {
++yylineno; ++yylineno;
yylval->assign(1, '\n');
return '\n'; return '\n';
} }
YY_BREAK YY_BREAK
...@@ -1052,7 +1113,17 @@ YY_RULE_SETUP ...@@ -1052,7 +1113,17 @@ YY_RULE_SETUP
} }
YY_BREAK YY_BREAK
case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(INITIAL):
{ yyterminate(); } case YY_STATE_EOF(COMMENT):
{
if (YY_START == COMMENT)
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno),
"");
}
yylval->clear();
yyterminate();
}
YY_BREAK YY_BREAK
case 37: case 37:
YY_RULE_SETUP YY_RULE_SETUP
......
...@@ -91,11 +91,6 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -91,11 +91,6 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
<COMMENT>[^*\r\n]+ <COMMENT>[^*\r\n]+
<COMMENT>"*" <COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; } <COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT><<EOF>> {
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno), "");
yyterminate();
}
<COMMENT>"*/" { <COMMENT>"*/" {
yyextra->leadingSpace = true; yyextra->leadingSpace = true;
BEGIN(INITIAL); BEGIN(INITIAL);
...@@ -104,6 +99,7 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -104,6 +99,7 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
# { # {
// # is only valid at start of line for preprocessor directives. // # is only valid at start of line for preprocessor directives.
if (yyextra->lineStart) { if (yyextra->lineStart) {
yylval->assign(1, yytext[0]);
return yytext[0]; return yytext[0];
} else { } else {
yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER, yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
...@@ -135,33 +131,101 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -135,33 +131,101 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
std::string(yytext, yyleng)); std::string(yytext, yyleng));
} }
"++" { return pp::Token::OP_INC; } "++" {
"--" { return pp::Token::OP_DEC; } yylval->assign(yytext, yyleng);
"<<" { return pp::Token::OP_LEFT; } return pp::Token::OP_INC;
">>" { return pp::Token::OP_RIGHT; } }
"<=" { return pp::Token::OP_LE; } "--" {
">=" { return pp::Token::OP_GE; } yylval->assign(yytext, yyleng);
"==" { return pp::Token::OP_EQ; } return pp::Token::OP_DEC;
"!=" { return pp::Token::OP_NE; } }
"&&" { return pp::Token::OP_AND; } "<<" {
"^^" { return pp::Token::OP_XOR; } yylval->assign(yytext, yyleng);
"||" { return pp::Token::OP_OR; } return pp::Token::OP_LEFT;
"+=" { return pp::Token::OP_ADD_ASSIGN; } }
"-=" { return pp::Token::OP_SUB_ASSIGN; } ">>" {
"*=" { return pp::Token::OP_MUL_ASSIGN; } yylval->assign(yytext, yyleng);
"/=" { return pp::Token::OP_DIV_ASSIGN; } return pp::Token::OP_RIGHT;
"%=" { return pp::Token::OP_MOD_ASSIGN; } }
"<<=" { return pp::Token::OP_LEFT_ASSIGN; } "<=" {
">>=" { return pp::Token::OP_RIGHT_ASSIGN; } yylval->assign(yytext, yyleng);
"&=" { return pp::Token::OP_AND_ASSIGN; } return pp::Token::OP_LE;
"^=" { return pp::Token::OP_XOR_ASSIGN; } }
"|=" { return pp::Token::OP_OR_ASSIGN; } ">=" {
yylval->assign(yytext, yyleng);
{PUNCTUATOR} { return yytext[0]; } return pp::Token::OP_GE;
}
"==" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_EQ;
}
"!=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_NE;
}
"&&" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND;
}
"^^" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR;
}
"||" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR;
}
"+=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_AND_ASSIGN;
}
"^=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
yylval->assign(yytext, yyleng);
return pp::Token::OP_OR_ASSIGN;
}
{PUNCTUATOR} {
yylval->assign(1, yytext[0]);
return yytext[0];
}
[ \t\v\f]+ { yyextra->leadingSpace = true; } [ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} { {NEWLINE} {
++yylineno; ++yylineno;
yylval->assign(1, '\n');
return '\n'; return '\n';
} }
...@@ -171,7 +235,16 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -171,7 +235,16 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
std::string(yytext, yyleng)); std::string(yytext, yyleng));
} }
<<EOF>> { yyterminate(); } <*><<EOF>> {
if (YY_START == COMMENT)
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno),
"");
}
yylval->clear();
yyterminate();
}
%% %%
......
...@@ -77,6 +77,7 @@ TEST_P(CharTest, Identified) ...@@ -77,6 +77,7 @@ TEST_P(CharTest, Identified)
else if (isPunctuator(str[0])) else if (isPunctuator(str[0]))
{ {
expectedType = str[0]; expectedType = str[0];
expectedValue = str;
} }
else if (isWhitespace(str[0])) else if (isWhitespace(str[0]))
{ {
......
...@@ -53,7 +53,7 @@ TEST(BlockComment, CommentReplacedWithSpace) ...@@ -53,7 +53,7 @@ TEST(BlockComment, CommentReplacedWithSpace)
pp::Token token; pp::Token token;
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("bar", token.value.c_str()); EXPECT_EQ("bar", token.value);
EXPECT_TRUE(token.hasLeadingSpace()); EXPECT_TRUE(token.hasLeadingSpace());
} }
......
...@@ -19,7 +19,7 @@ static void PreprocessAndVerifyIdentifier(const char* str) ...@@ -19,7 +19,7 @@ static void PreprocessAndVerifyIdentifier(const char* str)
pp::Token token; pp::Token token;
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ(str, token.value.c_str()); EXPECT_EQ(str, token.value);
} }
#if GTEST_HAS_COMBINE #if GTEST_HAS_COMBINE
......
...@@ -21,7 +21,7 @@ static void PreprocessAndVerifyLocation(int count, ...@@ -21,7 +21,7 @@ static void PreprocessAndVerifyLocation(int count,
pp::Token token; pp::Token token;
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("foo", token.value.c_str()); EXPECT_EQ("foo", token.value);
EXPECT_EQ(location.file, token.location.file); EXPECT_EQ(location.file, token.location.file);
EXPECT_EQ(location.line, token.location.line); EXPECT_EQ(location.line, token.location.line);
......
...@@ -60,7 +60,7 @@ TEST_P(IntegerTest, IntegerIdentified) ...@@ -60,7 +60,7 @@ TEST_P(IntegerTest, IntegerIdentified)
pp::Token token; pp::Token token;
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::CONST_INT, token.type); EXPECT_EQ(pp::Token::CONST_INT, token.type);
EXPECT_STREQ(cstr, token.value.c_str()); EXPECT_EQ(str, token.value);
} }
#define CLOSED_RANGE(x, y) testing::Range(x, static_cast<char>((y) + 1)) #define CLOSED_RANGE(x, y) testing::Range(x, static_cast<char>((y) + 1))
...@@ -99,7 +99,7 @@ static void PreprocessAndVerifyFloat(const char* str) ...@@ -99,7 +99,7 @@ static void PreprocessAndVerifyFloat(const char* str)
pp::Token token; pp::Token token;
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::CONST_FLOAT, token.type); EXPECT_EQ(pp::Token::CONST_FLOAT, token.type);
EXPECT_STREQ(str, token.value.c_str()); EXPECT_EQ(str, token.value);
} }
typedef std::tr1::tuple<char, char, const char*, char> FloatScientificParams; typedef std::tr1::tuple<char, char, const char*, char> FloatScientificParams;
......
...@@ -33,6 +33,7 @@ TEST_P(OperatorTest, Identified) ...@@ -33,6 +33,7 @@ TEST_P(OperatorTest, Identified)
pp::Token token; pp::Token token;
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(param.op, token.type); EXPECT_EQ(param.op, token.type);
EXPECT_EQ(param.str, token.value);
} }
static const OperatorTestParam kOperators[] = { static const OperatorTestParam kOperators[] = {
......
...@@ -39,7 +39,7 @@ TEST_P(SpaceCharTest, SpaceIgnored) ...@@ -39,7 +39,7 @@ TEST_P(SpaceCharTest, SpaceIgnored)
// Identifier "foo" is returned after ignoring the whitespace characters. // Identifier "foo" is returned after ignoring the whitespace characters.
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ(identifier, token.value.c_str()); EXPECT_EQ(identifier, token.value);
// The whitespace character is however recorded with the next token. // The whitespace character is however recorded with the next token.
EXPECT_TRUE(token.hasLeadingSpace()); EXPECT_TRUE(token.hasLeadingSpace());
} }
...@@ -78,7 +78,7 @@ TEST_P(SpaceStringTest, SpaceIgnored) ...@@ -78,7 +78,7 @@ TEST_P(SpaceStringTest, SpaceIgnored)
preprocessor.lex(&token); preprocessor.lex(&token);
// Identifier "foo" is returned after ignoring the whitespace characters. // Identifier "foo" is returned after ignoring the whitespace characters.
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ(identifier, token.value.c_str()); EXPECT_EQ(identifier, token.value);
// The whitespace character is however recorded with the next token. // The whitespace character is however recorded with the next token.
EXPECT_TRUE(token.hasLeadingSpace()); EXPECT_TRUE(token.hasLeadingSpace());
} }
...@@ -105,7 +105,7 @@ TEST(SpaceTest, LeadingSpace) ...@@ -105,7 +105,7 @@ TEST(SpaceTest, LeadingSpace)
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("foo", token.value.c_str()); EXPECT_EQ("foo", token.value);
EXPECT_TRUE(token.hasLeadingSpace()); EXPECT_TRUE(token.hasLeadingSpace());
preprocessor.lex(&token); preprocessor.lex(&token);
...@@ -118,6 +118,6 @@ TEST(SpaceTest, LeadingSpace) ...@@ -118,6 +118,6 @@ TEST(SpaceTest, LeadingSpace)
preprocessor.lex(&token); preprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type); EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_STREQ("bar", token.value.c_str()); EXPECT_EQ("bar", token.value);
EXPECT_FALSE(token.hasLeadingSpace()); EXPECT_FALSE(token.hasLeadingSpace());
} }
...@@ -14,7 +14,7 @@ TEST(TokenTest, DefaultConstructor) ...@@ -14,7 +14,7 @@ TEST(TokenTest, DefaultConstructor)
EXPECT_EQ(0, token.flags); EXPECT_EQ(0, token.flags);
EXPECT_EQ(0, token.location.line); EXPECT_EQ(0, token.location.line);
EXPECT_EQ(0, token.location.file); EXPECT_EQ(0, token.location.file);
EXPECT_STREQ("", token.value.c_str()); EXPECT_EQ("", token.value);
} }
TEST(TokenTest, Assignment) TEST(TokenTest, Assignment)
...@@ -31,7 +31,7 @@ TEST(TokenTest, Assignment) ...@@ -31,7 +31,7 @@ TEST(TokenTest, Assignment)
EXPECT_EQ(0, token.flags); EXPECT_EQ(0, token.flags);
EXPECT_EQ(0, token.location.line); EXPECT_EQ(0, token.location.line);
EXPECT_EQ(0, token.location.file); EXPECT_EQ(0, token.location.file);
EXPECT_STREQ("", token.value.c_str()); EXPECT_EQ("", token.value);
} }
TEST(TokenTest, Equals) TEST(TokenTest, Equals)
...@@ -79,11 +79,11 @@ TEST(TokenTest, Write) ...@@ -79,11 +79,11 @@ TEST(TokenTest, Write)
std::stringstream out1; std::stringstream out1;
out1 << token; out1 << token;
EXPECT_TRUE(out1.good()); EXPECT_TRUE(out1.good());
EXPECT_STREQ("foo", out1.str().c_str()); EXPECT_EQ("foo", out1.str());
token.setHasLeadingSpace(true); token.setHasLeadingSpace(true);
std::stringstream out2; std::stringstream out2;
out2 << token; out2 << token;
EXPECT_TRUE(out2.good()); EXPECT_TRUE(out2.good());
EXPECT_STREQ(" foo", out2.str().c_str()); EXPECT_EQ(" foo", out2.str());
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment