Added tests for token location.

src/compiler/preprocessor/new/pp.l

@@ -27,18 +27,40 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 #include "Lexer.h"
 #include "Token.h"
 
+// Token location (file and line) is encoded into a single int so that
+// we can use yylineno macro.
+
+const unsigned int kLocationLineSize = 16;  // in bits.
+const unsigned int kLocationLineMask = (1 << kLocationLineSize) - 1;
+
+static int location(int file, int line)
+{
+    return (file << kLocationLineSize) | (line & kLocationLineMask);
+}
+
+static int locationFile(int loc)
+{
+    return loc >> kLocationLineSize;
+}
+
+static int locationLine(int loc)
+{
+    return loc & kLocationLineMask;
+}
+
 typedef std::string YYSTYPE;
 typedef pp::Token::Location YYLTYPE;
 
-#define YY_USER_ACTION           \
-    do {                         \
-        yylloc->file = 0;        \
-        yylloc->line = yylineno; \
+#define YY_USER_ACTION                          \
+    do {                                        \
+        yylloc->file = locationFile(yylineno);  \
+        yylloc->line = locationLine(yylineno);  \
     } while(0);
 
 // Suppress the default implementation of YY_INPUT which generated
 // compiler warnings.
 #define YY_INPUT
+
 %}
 
 %option nounput never-interactive
@@ -139,10 +161,15 @@ int ppwrap(yyscan_t scanner)
 {
     pp::Input* input = yyget_extra(scanner);
 
+    int file = -1;
     // Delete the current buffer before switching to the next one.
     YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(input->buffer);
     if (buffer != NULL)
     {
+        // Save the current token location before they get lost when the
+        // current input buffer is deleted.
+        file = locationFile(yyget_lineno(scanner));
+
         yy_delete_buffer(buffer, scanner);
         input->buffer = NULL;
     }
@@ -157,7 +184,10 @@ int ppwrap(yyscan_t scanner)
     else
         buffer = yy_scan_bytes(input->string[index], length, scanner);
 
-    // TODO(alokp): Increment token location.
+    // Increment file number and reset line number.
+    // This can be done only after we have created a new input buffer.
+    yyset_lineno(location(file + 1, 1), scanner);
+
     input->index = index;
     input->buffer = buffer;
     return 0;

src/compiler/preprocessor/new/pp_lex.cpp

@@ -527,13 +527,34 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 #include "Lexer.h"
 #include "Token.h"
 
+// Token location (file and line) is encoded into a single int so that
+// we can use yylineno macro.
+
+const unsigned int kLocationLineSize = 16;  // in bits.
+const unsigned int kLocationLineMask = (1 << kLocationLineSize) - 1;
+
+static int location(int file, int line)
+{
+    return (file << kLocationLineSize) | (line & kLocationLineMask);
+}
+
+static int locationFile(int loc)
+{
+    return loc >> kLocationLineSize;
+}
+
+static int locationLine(int loc)
+{
+    return loc & kLocationLineMask;
+}
+
 typedef std::string YYSTYPE;
 typedef pp::Token::Location YYLTYPE;
 
-#define YY_USER_ACTION           \
-    do {                         \
-        yylloc->file = 0;        \
-        yylloc->line = yylineno; \
+#define YY_USER_ACTION                          \
+    do {                                        \
+        yylloc->file = locationFile(yylineno);  \
+        yylloc->line = locationLine(yylineno);  \
     } while(0);
 
 // Suppress the default implementation of YY_INPUT which generated
@@ -2170,10 +2191,15 @@ int ppwrap(yyscan_t scanner)
 {
     pp::Input* input = ppget_extra(scanner);
 
+    int file = -1;
     // Delete the current buffer before switching to the next one.
     YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(input->buffer);
     if (buffer != NULL)
     {
+        // Save the current token location before they get lost when the
+        // current input buffer is deleted.
+        file = locationFile(ppget_lineno(scanner));
+
         pp_delete_buffer(buffer,scanner);
         input->buffer = NULL;
     }
@@ -2188,7 +2214,10 @@ int ppwrap(yyscan_t scanner)
     else
         buffer = pp_scan_bytes(input->string[index],length,scanner);
 
-    // TODO(alokp): Increment token location.
+    // Increment file number and reset line number.
+    // This can be done only after we have created a new input buffer.
+    ppset_lineno(location(file + 1, 1),scanner);
+
     input->index = index;
     input->buffer = buffer;
     return 0;

tests/build_tests.gyp

@@ -30,6 +30,7 @@
         '../third_party/googletest/src/gtest_main.cc',
         'preprocessor_tests/char_test.cpp',
         'preprocessor_tests/identifier_test.cpp',
+        'preprocessor_tests/location_test.cpp',
         'preprocessor_tests/number_test.cpp',
         'preprocessor_tests/token_test.cpp',
         'preprocessor_tests/space_test.cpp',

tests/preprocessor_tests/location_test.cpp

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "gtest/gtest.h"
+#include "Preprocessor.h"
+#include "Token.h"
+
+static void PreprocessAndVerifyLocation(int count,
+                                        const char* const string[],
+                                        const int length[],
+                                        pp::Token::Location location)
+{
+    pp::Token token;
+    pp::Preprocessor preprocessor;
+    ASSERT_TRUE(preprocessor.init(count, string, length));
+    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+    EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
+    EXPECT_STREQ("foo", token.value.c_str());
+
+    EXPECT_EQ(location.file, token.location.file);
+    EXPECT_EQ(location.line, token.location.line);
+}
+
+TEST(LocationTest, String0_Line1)
+{
+    const char* str = "foo";
+    pp::Token::Location loc;
+    loc.file = 0;
+    loc.line = 1;
+
+    SCOPED_TRACE("String0_Line1");
+    PreprocessAndVerifyLocation(1, &str, 0, loc);
+}
+
+TEST(LocationTest, String0_Line2)
+{
+    const char* str = "\nfoo";
+    pp::Token::Location loc;
+    loc.file = 0;
+    loc.line = 2;
+
+    SCOPED_TRACE("String0_Line2");
+    PreprocessAndVerifyLocation(1, &str, 0, loc);
+}
+
+TEST(LocationTest, String1_Line1)
+{
+    const char* const str[] = {"\n\n", "foo"};
+    pp::Token::Location loc;
+    loc.file = 1;
+    loc.line = 1;
+
+    SCOPED_TRACE("String1_Line1");
+    PreprocessAndVerifyLocation(2, str, 0, loc);
+}
+
+TEST(LocationTest, String1_Line2)
+{
+    const char* const str[] = {"\n\n", "\nfoo"};
+    pp::Token::Location loc;
+    loc.file = 1;
+    loc.line = 2;
+
+    SCOPED_TRACE("String1_Line2");
+    PreprocessAndVerifyLocation(2, str, 0, loc);
+}
+
+TEST(LocationTest, NewlineInsideCommentCounted)
+{
+    const char* str = "/*\n\n*/foo";
+    pp::Token::Location loc;
+    loc.file = 0;
+    loc.line = 3;
+
+    SCOPED_TRACE("NewlineInsideCommentCounted");
+    PreprocessAndVerifyLocation(1, &str, 0, loc);
+}
+
+// The location of a token straddling two or more strings is that of the
+// first character of the token.
+
+TEST(LocationTest, TokenStraddlingTwoStrings)
+{
+    const char* const str[] = {"f", "oo"};
+    pp::Token::Location loc;
+    loc.file = 0;
+    loc.line = 1;
+
+    SCOPED_TRACE("TokenStraddlingTwoStrings");
+    PreprocessAndVerifyLocation(2, str, 0, loc);
+}
+
+TEST(LocationTest, TokenStraddlingThreeStrings)
+{
+    const char* const str[] = {"f", "o", "o"};
+    pp::Token::Location loc;
+    loc.file = 0;
+    loc.line = 1;
+
+    SCOPED_TRACE("TokenStraddlingThreeStrings");
+    PreprocessAndVerifyLocation(3, str, 0, loc);
+}
+
+// TODO(alokp): Add tests for #line directives.