Token::value now contains text for all types of tokens - not just CONST_INT,…

src/compiler/preprocessor/new/Tokenizer.cpp

@@ -885,13 +885,6 @@ case 5:
 YY_RULE_SETUP
 { ++yylineno; }
 	YY_BREAK
-case YY_STATE_EOF(COMMENT):
-{
-    yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
-        pp::SourceLocation(yyfileno, yylineno), "");
-    yyterminate();
-}
-	YY_BREAK
 case 6:
 YY_RULE_SETUP
 {
@@ -904,6 +897,7 @@ YY_RULE_SETUP
 {
     // # is only valid at start of line for preprocessor directives.
     if (yyextra->lineStart) {
+        yylval->assign(1, yytext[0]);
         return yytext[0];
     } else {
         yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
@@ -945,91 +939,157 @@ YY_RULE_SETUP
 	YY_BREAK
 case 12:
 YY_RULE_SETUP
-{ return pp::Token::OP_INC; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_INC;
+}
 	YY_BREAK
 case 13:
 YY_RULE_SETUP
-{ return pp::Token::OP_DEC; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_DEC;
+}
 	YY_BREAK
 case 14:
 YY_RULE_SETUP
-{ return pp::Token::OP_LEFT; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_LEFT;
+}
 	YY_BREAK
 case 15:
 YY_RULE_SETUP
-{ return pp::Token::OP_RIGHT; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_RIGHT;
+}
 	YY_BREAK
 case 16:
 YY_RULE_SETUP
-{ return pp::Token::OP_LE; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_LE;
+}
 	YY_BREAK
 case 17:
 YY_RULE_SETUP
-{ return pp::Token::OP_GE; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_GE;
+}
 	YY_BREAK
 case 18:
 YY_RULE_SETUP
-{ return pp::Token::OP_EQ; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_EQ;
+}
 	YY_BREAK
 case 19:
 YY_RULE_SETUP
-{ return pp::Token::OP_NE; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_NE;
+}
 	YY_BREAK
 case 20:
 YY_RULE_SETUP
-{ return pp::Token::OP_AND; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_AND;
+}
 	YY_BREAK
 case 21:
 YY_RULE_SETUP
-{ return pp::Token::OP_XOR; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_XOR;
+}
 	YY_BREAK
 case 22:
 YY_RULE_SETUP
-{ return pp::Token::OP_OR; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_OR;
+}
 	YY_BREAK
 case 23:
 YY_RULE_SETUP
-{  return pp::Token::OP_ADD_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_ADD_ASSIGN;
+}
 	YY_BREAK
 case 24:
 YY_RULE_SETUP
-{  return pp::Token::OP_SUB_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_SUB_ASSIGN;
+}
 	YY_BREAK
 case 25:
 YY_RULE_SETUP
-{  return pp::Token::OP_MUL_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_MUL_ASSIGN;
+}
 	YY_BREAK
 case 26:
 YY_RULE_SETUP
-{  return pp::Token::OP_DIV_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_DIV_ASSIGN;
+}
 	YY_BREAK
 case 27:
 YY_RULE_SETUP
-{  return pp::Token::OP_MOD_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_MOD_ASSIGN;
+}
 	YY_BREAK
 case 28:
 YY_RULE_SETUP
-{  return pp::Token::OP_LEFT_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_LEFT_ASSIGN;
+}
 	YY_BREAK
 case 29:
 YY_RULE_SETUP
-{  return pp::Token::OP_RIGHT_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_RIGHT_ASSIGN;
+}
 	YY_BREAK
 case 30:
 YY_RULE_SETUP
-{  return pp::Token::OP_AND_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_AND_ASSIGN;
+}
 	YY_BREAK
 case 31:
 YY_RULE_SETUP
-{  return pp::Token::OP_XOR_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_XOR_ASSIGN;
+}
 	YY_BREAK
 case 32:
 YY_RULE_SETUP
-{  return pp::Token::OP_OR_ASSIGN; }
+{
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_OR_ASSIGN;
+}
 	YY_BREAK
 case 33:
 YY_RULE_SETUP
-{ return yytext[0]; }
+{
+    yylval->assign(1, yytext[0]);
+    return yytext[0];
+}
 	YY_BREAK
 case 34:
 YY_RULE_SETUP
@@ -1040,6 +1100,7 @@ case 35:
 YY_RULE_SETUP
 {
     ++yylineno;
+    yylval->assign(1, '\n');
     return '\n';
 }
 	YY_BREAK
@@ -1052,7 +1113,17 @@ YY_RULE_SETUP
 }
 	YY_BREAK
 case YY_STATE_EOF(INITIAL):
-{ yyterminate(); }
+case YY_STATE_EOF(COMMENT):
+{
+    if (YY_START == COMMENT)
+    {
+        yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+                                     pp::SourceLocation(yyfileno, yylineno),
+                                     "");
+    }
+    yylval->clear();
+    yyterminate();
+}
 	YY_BREAK
 case 37:
 YY_RULE_SETUP

src/compiler/preprocessor/new/Tokenizer.l

@@ -91,11 +91,6 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 <COMMENT>[^*\r\n]+
 <COMMENT>"*"
 <COMMENT>{NEWLINE} { ++yylineno; }
-<COMMENT><<EOF>> {
-    yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
-        pp::SourceLocation(yyfileno, yylineno), "");
-    yyterminate();
-}
 <COMMENT>"*/" {
     yyextra->leadingSpace = true;
     BEGIN(INITIAL);
@@ -104,6 +99,7 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 # {
     // # is only valid at start of line for preprocessor directives.
     if (yyextra->lineStart) {
+        yylval->assign(1, yytext[0]);
         return yytext[0];
     } else {
         yyextra->diagnostics->report(pp::Diagnostics::INVALID_CHARACTER,
@@ -135,33 +131,101 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
                                  std::string(yytext, yyleng));
 }
 
-"++"  { return pp::Token::OP_INC; }
-"--"  { return pp::Token::OP_DEC; }
-"<<"  { return pp::Token::OP_LEFT; }
-">>"  { return pp::Token::OP_RIGHT; }
-"<="  { return pp::Token::OP_LE; }
-">="  { return pp::Token::OP_GE; }
-"=="  { return pp::Token::OP_EQ; }
-"!="  { return pp::Token::OP_NE; }
-"&&"  { return pp::Token::OP_AND; }
-"^^"  { return pp::Token::OP_XOR; }
-"||"  { return pp::Token::OP_OR; }
-"+="  {  return pp::Token::OP_ADD_ASSIGN; }
-"-="  {  return pp::Token::OP_SUB_ASSIGN; }
-"*="  {  return pp::Token::OP_MUL_ASSIGN; }
-"/="  {  return pp::Token::OP_DIV_ASSIGN; }
-"%="  {  return pp::Token::OP_MOD_ASSIGN; }
-"<<=" {  return pp::Token::OP_LEFT_ASSIGN; }
-">>=" {  return pp::Token::OP_RIGHT_ASSIGN; }
-"&="  {  return pp::Token::OP_AND_ASSIGN; }
-"^="  {  return pp::Token::OP_XOR_ASSIGN; }
-"|="  {  return pp::Token::OP_OR_ASSIGN; }
-
-{PUNCTUATOR} { return yytext[0]; }
+"++" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_INC;
+}
+"--" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_DEC;
+}
+"<<" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_LEFT;
+}
+">>" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_RIGHT;
+}
+"<=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_LE;
+}
+">=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_GE;
+}
+"==" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_EQ;
+}
+"!=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_NE;
+}
+"&&" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_AND;
+}
+"^^" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_XOR;
+}
+"||" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_OR;
+}
+"+=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_ADD_ASSIGN;
+}
+"-=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_SUB_ASSIGN;
+}
+"*=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_MUL_ASSIGN;
+}
+"/=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_DIV_ASSIGN;
+}
+"%=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_MOD_ASSIGN;
+}
+"<<=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_LEFT_ASSIGN;
+}
+">>=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_RIGHT_ASSIGN;
+}
+"&=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_AND_ASSIGN;
+}
+"^=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_XOR_ASSIGN;
+}
+"|=" {
+    yylval->assign(yytext, yyleng);
+    return pp::Token::OP_OR_ASSIGN;
+}
+
+{PUNCTUATOR} {
+    yylval->assign(1, yytext[0]);
+    return yytext[0];
+}
+
 [ \t\v\f]+   { yyextra->leadingSpace = true; }
 
 {NEWLINE} {
     ++yylineno;
+    yylval->assign(1, '\n');
     return '\n';
 }
 
@@ -171,7 +235,16 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
                                  std::string(yytext, yyleng));
 }
 
-<<EOF>>   { yyterminate(); }
+<*><<EOF>> {
+    if (YY_START == COMMENT)
+    {
+        yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
+                                     pp::SourceLocation(yyfileno, yylineno),
+                                     "");
+    }
+    yylval->clear();
+    yyterminate();
+}
 
 %%
 

tests/preprocessor_tests/char_test.cpp

@@ -77,6 +77,7 @@ TEST_P(CharTest, Identified)
     else if (isPunctuator(str[0]))
     {
         expectedType = str[0];
+        expectedValue = str;
     }
     else if (isWhitespace(str[0]))
     {

tests/preprocessor_tests/comment_test.cpp

@@ -53,7 +53,7 @@ TEST(BlockComment, CommentReplacedWithSpace)
     pp::Token token;
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ("bar", token.value.c_str());
+    EXPECT_EQ("bar", token.value);
     EXPECT_TRUE(token.hasLeadingSpace());
 }
 

tests/preprocessor_tests/identifier_test.cpp

@@ -19,7 +19,7 @@ static void PreprocessAndVerifyIdentifier(const char* str)
     pp::Token token;
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ(str, token.value.c_str());
+    EXPECT_EQ(str, token.value);
 }
 
 #if GTEST_HAS_COMBINE

tests/preprocessor_tests/location_test.cpp

@@ -21,7 +21,7 @@ static void PreprocessAndVerifyLocation(int count,
     pp::Token token;
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ("foo", token.value.c_str());
+    EXPECT_EQ("foo", token.value);
 
     EXPECT_EQ(location.file, token.location.file);
     EXPECT_EQ(location.line, token.location.line);

tests/preprocessor_tests/number_test.cpp

@@ -60,7 +60,7 @@ TEST_P(IntegerTest, IntegerIdentified)
     pp::Token token;
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::CONST_INT, token.type);
-    EXPECT_STREQ(cstr, token.value.c_str());
+    EXPECT_EQ(str, token.value);
 }
 
 #define CLOSED_RANGE(x, y) testing::Range(x, static_cast<char>((y) + 1))
@@ -99,7 +99,7 @@ static void PreprocessAndVerifyFloat(const char* str)
     pp::Token token;
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::CONST_FLOAT, token.type);
-    EXPECT_STREQ(str, token.value.c_str());
+    EXPECT_EQ(str, token.value);
 }
 
 typedef std::tr1::tuple<char, char, const char*, char> FloatScientificParams;

tests/preprocessor_tests/operator_test.cpp

@@ -33,6 +33,7 @@ TEST_P(OperatorTest, Identified)
     pp::Token token;
     preprocessor.lex(&token);
     EXPECT_EQ(param.op, token.type);
+    EXPECT_EQ(param.str, token.value);
 }
 
 static const OperatorTestParam kOperators[] = {

tests/preprocessor_tests/space_test.cpp

@@ -39,7 +39,7 @@ TEST_P(SpaceCharTest, SpaceIgnored)
     // Identifier "foo" is returned after ignoring the whitespace characters.
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ(identifier, token.value.c_str());
+    EXPECT_EQ(identifier, token.value);
     // The whitespace character is however recorded with the next token.
     EXPECT_TRUE(token.hasLeadingSpace());
 }
@@ -78,7 +78,7 @@ TEST_P(SpaceStringTest, SpaceIgnored)
     preprocessor.lex(&token);
     // Identifier "foo" is returned after ignoring the whitespace characters.
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ(identifier, token.value.c_str());
+    EXPECT_EQ(identifier, token.value);
     // The whitespace character is however recorded with the next token.
     EXPECT_TRUE(token.hasLeadingSpace());
 }
@@ -105,7 +105,7 @@ TEST(SpaceTest, LeadingSpace)
 
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ("foo", token.value.c_str());
+    EXPECT_EQ("foo", token.value);
     EXPECT_TRUE(token.hasLeadingSpace());
 
     preprocessor.lex(&token);
@@ -118,6 +118,6 @@ TEST(SpaceTest, LeadingSpace)
 
     preprocessor.lex(&token);
     EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
-    EXPECT_STREQ("bar", token.value.c_str());
+    EXPECT_EQ("bar", token.value);
     EXPECT_FALSE(token.hasLeadingSpace());
 }

tests/preprocessor_tests/token_test.cpp

@@ -14,7 +14,7 @@ TEST(TokenTest, DefaultConstructor)
     EXPECT_EQ(0, token.flags);
     EXPECT_EQ(0, token.location.line);
     EXPECT_EQ(0, token.location.file);
-    EXPECT_STREQ("", token.value.c_str());
+    EXPECT_EQ("", token.value);
 }
 
 TEST(TokenTest, Assignment)
@@ -31,7 +31,7 @@ TEST(TokenTest, Assignment)
     EXPECT_EQ(0, token.flags);
     EXPECT_EQ(0, token.location.line);
     EXPECT_EQ(0, token.location.file);
-    EXPECT_STREQ("", token.value.c_str());
+    EXPECT_EQ("", token.value);
 }
 
 TEST(TokenTest, Equals)
@@ -79,11 +79,11 @@ TEST(TokenTest, Write)
     std::stringstream out1;
     out1 << token;
     EXPECT_TRUE(out1.good());
-    EXPECT_STREQ("foo", out1.str().c_str());
+    EXPECT_EQ("foo", out1.str());
 
     token.setHasLeadingSpace(true);
     std::stringstream out2;
     out2 << token;
     EXPECT_TRUE(out2.good());
-    EXPECT_STREQ(" foo", out2.str().c_str());
+    EXPECT_EQ(" foo", out2.str());
 }