Changes to handle comments properly and associated tests.

src/compiler/preprocessor/new/Token.h

@@ -20,6 +20,7 @@ struct Token
         // Token IDs for error conditions are negative.
         INVALID_CHARACTER = -1,
         INVALID_NUMBER = -2,
+        EOF_IN_COMMENT = -3,
 
         // Indicates EOF.
         LAST = 0,

src/compiler/preprocessor/new/pp.l

@@ -87,12 +87,14 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 "//"[^\r\n]*
 
     /* Block comment */
+    /* Line breaks are just counted - not returned. */
+    /* The comment is replaced by a single space. */ 
 "/*"                { BEGIN(COMMENT); }
-<COMMENT>[^*\r\n]*
-<COMMENT>[^*\r\n]*{NEWLINE}       { ++yylineno; return '\n'; }
-<COMMENT>"*"+[^*/\r\n]*
-<COMMENT>"*"+[^*/\r\n]*{NEWLINE}  { ++yylineno; return '\n'; }
-<COMMENT>"*"+"/"                  { BEGIN(INITIAL); return ' '; }
+<COMMENT>[^*\r\n]+
+<COMMENT>"*"
+<COMMENT>{NEWLINE}  { ++yylineno; }
+<COMMENT><<EOF>>    { return pp::Token::EOF_IN_COMMENT; }
+<COMMENT>"*/"       { BEGIN(INITIAL); return ' '; }
 
 # { return yytext[0]; }
 
@@ -153,7 +155,7 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
     return pp::Token::INVALID_CHARACTER;
 }
 
-<*><<EOF>>   { yyterminate(); }
+<<EOF>>   { yyterminate(); }
 
 %%
 

tests/build_tests.gyp

@@ -29,6 +29,7 @@
       'sources': [
         '../third_party/googletest/src/gtest_main.cc',
         'preprocessor_tests/char_test.cpp',
+        'preprocessor_tests/comment_test.cpp',
         'preprocessor_tests/identifier_test.cpp',
         'preprocessor_tests/location_test.cpp',
         'preprocessor_tests/number_test.cpp',

tests/preprocessor_tests/comment_test.cpp

+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "gtest/gtest.h"
+#include "Preprocessor.h"
+#include "Token.h"
+
+class CommentTest : public testing::TestWithParam<const char*>
+{
+};
+
+TEST_P(CommentTest, CommentIgnored)
+{
+    const char* str = GetParam();
+
+    pp::Token token;
+    pp::Preprocessor preprocessor;
+    ASSERT_TRUE(preprocessor.init(1, &str, 0));
+    EXPECT_EQ(pp::Token::LAST, preprocessor.lex(&token));
+    EXPECT_EQ(pp::Token::LAST, token.type);
+}
+
+INSTANTIATE_TEST_CASE_P(LineComment, CommentTest,
+                        testing::Values("//foo\n", // With newline.
+                                        "//foo",   // Without newline.
+                                        "//**/",   // Nested block comment.
+                                        "////",    // Nested line comment.
+                                        "//\""));  // Invalid character.  
+
+INSTANTIATE_TEST_CASE_P(BlockComment, CommentTest,
+                        testing::Values("/*foo*/",
+                                        "/*foo\n*/", // With newline.
+                                        "/*//*/",    // Nested line comment.
+                                        "/*/**/",    // Nested block comment.
+                                        "/***/",     // With lone '*'.
+                                        "/*\"*/"));  // Invalid character.
+
+TEST(BlockComment, CommentReplacedWithSpace)
+{
+    const char* str = "/*foo*/bar";
+
+    pp::Token token;
+    pp::Preprocessor preprocessor;
+    ASSERT_TRUE(preprocessor.init(1, &str, 0));
+    EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+    EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
+    EXPECT_STREQ("bar", token.value.c_str());
+    EXPECT_TRUE(token.hasLeadingSpace());
+}
+
+TEST(BlockComment, UnterminatedComment)
+{
+    const char* str = "/*foo";
+
+    pp::Token token;
+    pp::Preprocessor preprocessor;
+    ASSERT_TRUE(preprocessor.init(1, &str, 0));
+    EXPECT_EQ(pp::Token::EOF_IN_COMMENT, preprocessor.lex(&token));
+    EXPECT_EQ(pp::Token::EOF_IN_COMMENT, token.type);
+}

tests/preprocessor_tests/location_test.cpp

@@ -79,6 +79,21 @@ TEST(LocationTest, NewlineInsideCommentCounted)
     PreprocessAndVerifyLocation(1, &str, 0, loc);
 }
 
+TEST(LocationTest, ErrorLocationAfterComment)
+{
+    const char* str = "/*\n\n*/@";
+
+    pp::Token token;
+    pp::Preprocessor preprocessor;
+    ASSERT_TRUE(preprocessor.init(1, &str, 0));
+    EXPECT_EQ(pp::Token::INVALID_CHARACTER, preprocessor.lex(&token));
+    EXPECT_EQ(pp::Token::INVALID_CHARACTER, token.type);
+    EXPECT_STREQ("@", token.value.c_str());
+
+    EXPECT_EQ(0, token.location.file);
+    EXPECT_EQ(3, token.location.line);
+}
+
 // The location of a token straddling two or more strings is that of the
 // first character of the token.