Fixed the location of EOF token. Added three new location tests for EOF.

src/compiler/preprocessor/new/Token.cpp

@@ -9,6 +9,38 @@
 namespace pp
 {
 
+void Token::reset()
+{
+    type = 0;
+    flags = 0;
+    location = SourceLocation();
+    value.clear();
+}
+
+bool Token::equals(const Token& other) const
+{
+    return (type == other.type) &&
+           (flags == other.flags) &&
+           (location == other.location) &&
+           (value == other.value);
+}
+
+void Token::setAtStartOfLine(bool start)
+{
+    if (start)
+        flags |= AT_START_OF_LINE;
+    else
+        flags &= ~AT_START_OF_LINE;
+}
+
+void Token::setHasLeadingSpace(bool space)
+{
+    if (space)
+        flags |= HAS_LEADING_SPACE;
+    else
+        flags &= ~HAS_LEADING_SPACE;
+}
+
 std::ostream& operator<<(std::ostream& out, const Token& token)
 {
     if (token.hasLeadingSpace())

src/compiler/preprocessor/new/Token.h

@@ -50,35 +50,22 @@ struct Token
     };
     enum Flags
     {
-        HAS_LEADING_SPACE = 1 << 0
+        AT_START_OF_LINE  = 1 << 0,
+        HAS_LEADING_SPACE = 1 << 1
     };
 
     Token() : type(0), flags(0) { }
 
-    void reset()
-    {
-        type = 0;
-        flags = 0;
-        location = SourceLocation();
-        value.clear();
-    }
+    void reset();
+    bool equals(const Token& other) const;
 
-    bool equals(const Token& other) const
-    {
-        return (type == other.type) &&
-               (flags == other.flags) &&
-               (location == other.location) &&
-               (value == other.value);
-    }
+    // Returns true if this is the first token on line.
+    // It disregards any leading whitespace.
+    bool atStartOfLine() const { return (flags & AT_START_OF_LINE) != 0; }
+    void setAtStartOfLine(bool start);
 
     bool hasLeadingSpace() const { return (flags & HAS_LEADING_SPACE) != 0; }
-    void setHasLeadingSpace(bool space)
-    {
-        if (space)
-            flags |= HAS_LEADING_SPACE;
-        else
-            flags &= ~HAS_LEADING_SPACE;
-    }
+    void setHasLeadingSpace(bool space);
 
     int type;
     int flags;

src/compiler/preprocessor/new/Tokenizer.cpp

@@ -533,18 +533,20 @@ typedef pp::SourceLocation YYLTYPE;
         yyextra->lineStart = true;     \
     } while(0);
 
-#define YY_USER_ACTION                                            \
-    do {                                                          \
-        pp::Input* input = &yyextra->input;                       \
-        pp::Input::Location* scanLoc = &yyextra->scanLoc;         \
-        while (scanLoc->cIndex >= input->length(scanLoc->sIndex)) \
-        {                                                         \
-            scanLoc->cIndex -= input->length(scanLoc->sIndex++);  \
-            ++yyfileno; yylineno = 1;                             \
-        }                                                         \
-        yylloc->file = yyfileno;                                  \
-        yylloc->line = yylineno;                                  \
-        scanLoc->cIndex += yyleng;                                \
+#define YY_USER_ACTION                                              \
+    do                                                              \
+    {                                                               \
+        pp::Input* input = &yyextra->input;                         \
+        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
+        while ((scanLoc->sIndex < input->count()) &&                \
+               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
+        {                                                           \
+            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
+            ++yyfileno; yylineno = 1;                               \
+        }                                                           \
+        yylloc->file = yyfileno;                                    \
+        yylloc->line = yylineno;                                    \
+        scanLoc->cIndex += yyleng;                                  \
     } while(0);
 
 #define YY_INPUT(buf, result, maxSize) \
@@ -1115,13 +1117,28 @@ YY_RULE_SETUP
 case YY_STATE_EOF(INITIAL):
 case YY_STATE_EOF(COMMENT):
 {
+    // YY_USER_ACTION is not invoked for handling EOF.
+    // Set the location for EOF token manually.
+    pp::Input* input = &yyextra->input;
+    pp::Input::Location* scanLoc = &yyextra->scanLoc;
+    int sIndexMax = std::max(0, input->count() - 1);
+    if (scanLoc->sIndex != sIndexMax)
+    {
+        // We can only reach here if there are empty strings at the
+        // end of the input.
+        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
+        yyfileno = sIndexMax; yylineno = 1;
+    }
+    yylloc->file = yyfileno;
+    yylloc->line = yylineno;
+    yylval->clear();
+
     if (YY_START == COMMENT)
     {
         yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
                                      pp::SourceLocation(yyfileno, yylineno),
                                      "");
     }
-    yylval->clear();
     yyterminate();
 }
 	YY_BREAK
@@ -2289,9 +2306,11 @@ void Tokenizer::lex(Token* token)
 {
     token->type = pplex(&token->value,&token->location,mHandle);
 
+    token->setAtStartOfLine(mContext.lineStart);
+    mContext.lineStart = token->type == '\n';
+
     token->setHasLeadingSpace(mContext.leadingSpace);
     mContext.leadingSpace = false;
-    mContext.lineStart = token->type == '\n';
 }
 
 bool Tokenizer::initScanner()

src/compiler/preprocessor/new/Tokenizer.l

@@ -42,18 +42,20 @@ typedef pp::SourceLocation YYLTYPE;
         yyextra->lineStart = true;     \
     } while(0);
 
-#define YY_USER_ACTION                                            \
-    do {                                                          \
-        pp::Input* input = &yyextra->input;                       \
-        pp::Input::Location* scanLoc = &yyextra->scanLoc;         \
-        while (scanLoc->cIndex >= input->length(scanLoc->sIndex)) \
-        {                                                         \
-            scanLoc->cIndex -= input->length(scanLoc->sIndex++);  \
-            ++yyfileno; yylineno = 1;                             \
-        }                                                         \
-        yylloc->file = yyfileno;                                  \
-        yylloc->line = yylineno;                                  \
-        scanLoc->cIndex += yyleng;                                \
+#define YY_USER_ACTION                                              \
+    do                                                              \
+    {                                                               \
+        pp::Input* input = &yyextra->input;                         \
+        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
+        while ((scanLoc->sIndex < input->count()) &&                \
+               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
+        {                                                           \
+            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
+            ++yyfileno; yylineno = 1;                               \
+        }                                                           \
+        yylloc->file = yyfileno;                                    \
+        yylloc->line = yylineno;                                    \
+        scanLoc->cIndex += yyleng;                                  \
     } while(0);
 
 #define YY_INPUT(buf, result, maxSize) \
@@ -236,13 +238,28 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 }
 
 <*><<EOF>> {
+    // YY_USER_ACTION is not invoked for handling EOF.
+    // Set the location for EOF token manually.
+    pp::Input* input = &yyextra->input;
+    pp::Input::Location* scanLoc = &yyextra->scanLoc;
+    int sIndexMax = std::max(0, input->count() - 1);
+    if (scanLoc->sIndex != sIndexMax)
+    {
+        // We can only reach here if there are empty strings at the
+        // end of the input.
+        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
+        yyfileno = sIndexMax; yylineno = 1;
+    }
+    yylloc->file = yyfileno;
+    yylloc->line = yylineno;
+    yylval->clear();
+
     if (YY_START == COMMENT)
     {
         yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
                                      pp::SourceLocation(yyfileno, yylineno),
                                      "");
     }
-    yylval->clear();
     yyterminate();
 }
 
@@ -273,9 +290,11 @@ void Tokenizer::lex(Token* token)
 {
     token->type = yylex(&token->value, &token->location, mHandle);
 
+    token->setAtStartOfLine(mContext.lineStart);
+    mContext.lineStart = token->type == '\n';
+
     token->setHasLeadingSpace(mContext.leadingSpace);
     mContext.leadingSpace = false;
-    mContext.lineStart = token->type == '\n';
 }
 
 bool Tokenizer::initScanner()

tests/preprocessor_tests/location_test.cpp

@@ -35,7 +35,7 @@ TEST_F(LocationTest, String0_Line1)
     loc.line = 1;
 
     SCOPED_TRACE("String0_Line1");
-    preprocess(1, &str, 0, loc);
+    preprocess(1, &str, NULL, loc);
 }
 
 TEST_F(LocationTest, String0_Line2)
@@ -46,7 +46,7 @@ TEST_F(LocationTest, String0_Line2)
     loc.line = 2;
 
     SCOPED_TRACE("String0_Line2");
-    preprocess(1, &str, 0, loc);
+    preprocess(1, &str, NULL, loc);
 }
 
 TEST_F(LocationTest, String1_Line1)
@@ -57,7 +57,7 @@ TEST_F(LocationTest, String1_Line1)
     loc.line = 1;
 
     SCOPED_TRACE("String1_Line1");
-    preprocess(2, str, 0, loc);
+    preprocess(2, str, NULL, loc);
 }
 
 TEST_F(LocationTest, String1_Line2)
@@ -68,7 +68,7 @@ TEST_F(LocationTest, String1_Line2)
     loc.line = 2;
 
     SCOPED_TRACE("String1_Line2");
-    preprocess(2, str, 0, loc);
+    preprocess(2, str, NULL, loc);
 }
 
 TEST_F(LocationTest, NewlineInsideCommentCounted)
@@ -79,14 +79,14 @@ TEST_F(LocationTest, NewlineInsideCommentCounted)
     loc.line = 3;
 
     SCOPED_TRACE("NewlineInsideCommentCounted");
-    preprocess(1, &str, 0, loc);
+    preprocess(1, &str, NULL, loc);
 }
 
 TEST_F(LocationTest, ErrorLocationAfterComment)
 {
     const char* str = "/*\n\n*/@";
 
-    ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
+    ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
     EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::INVALID_CHARACTER,
                                     pp::SourceLocation(0, 3),
                                     "@"));
@@ -106,7 +106,7 @@ TEST_F(LocationTest, TokenStraddlingTwoStrings)
     loc.line = 1;
 
     SCOPED_TRACE("TokenStraddlingTwoStrings");
-    preprocess(2, str, 0, loc);
+    preprocess(2, str, NULL, loc);
 }
 
 TEST_F(LocationTest, TokenStraddlingThreeStrings)
@@ -117,7 +117,61 @@ TEST_F(LocationTest, TokenStraddlingThreeStrings)
     loc.line = 1;
 
     SCOPED_TRACE("TokenStraddlingThreeStrings");
-    preprocess(3, str, 0, loc);
+    preprocess(3, str, NULL, loc);
+}
+
+TEST_F(LocationTest, EndOfFileWithoutNewline)
+{
+    const char* const str[] = {"foo"};
+    ASSERT_TRUE(mPreprocessor.init(1, str, NULL));
+
+    pp::Token token;
+    mPreprocessor.lex(&token);
+    EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
+    EXPECT_EQ("foo", token.value);
+    EXPECT_EQ(0, token.location.file);
+    EXPECT_EQ(1, token.location.line);
+
+    mPreprocessor.lex(&token);
+    EXPECT_EQ(pp::Token::LAST, token.type);
+    EXPECT_EQ(0, token.location.file);
+    EXPECT_EQ(1, token.location.line);
+}
+
+TEST_F(LocationTest, EndOfFileAfterNewline)
+{
+    const char* const str[] = {"foo\n"};
+    ASSERT_TRUE(mPreprocessor.init(1, str, NULL));
+
+    pp::Token token;
+    mPreprocessor.lex(&token);
+    EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
+    EXPECT_EQ("foo", token.value);
+    EXPECT_EQ(0, token.location.file);
+    EXPECT_EQ(1, token.location.line);
+
+    mPreprocessor.lex(&token);
+    EXPECT_EQ(pp::Token::LAST, token.type);
+    EXPECT_EQ(0, token.location.file);
+    EXPECT_EQ(2, token.location.line);
+}
+
+TEST_F(LocationTest, EndOfFileAfterEmptyString)
+{
+    const char* const str[] = {"foo\n", "\n", ""};
+    ASSERT_TRUE(mPreprocessor.init(3, str, NULL));
+
+    pp::Token token;
+    mPreprocessor.lex(&token);
+    EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
+    EXPECT_EQ("foo", token.value);
+    EXPECT_EQ(0, token.location.file);
+    EXPECT_EQ(1, token.location.line);
+
+    mPreprocessor.lex(&token);
+    EXPECT_EQ(pp::Token::LAST, token.type);
+    EXPECT_EQ(2, token.location.file);
+    EXPECT_EQ(1, token.location.line);
 }
 
 // TODO(alokp): Add tests for #line directives.