Skip to content

A
angle
Commit 19d7aa60 by alokp@chromium.org
Options

Fixed the location of EOF token. Added three new location tests for EOF.

git-svn-id: https://angleproject.googlecode.com/svn/trunk@1125 736b8ea6-26fd-11df-bfd4-992fa37f6226
parent ddbb45d6
Showing with 169 additions and 58 deletions
src/compiler/preprocessor/new/Token.cpp
......@@ -9,6 +9,38 @@
namespace pp
{
void Token::reset()
{
type = 0;
flags = 0;
location = SourceLocation();
value.clear();
}
bool Token::equals(const Token& other) const
{
return (type == other.type) &&
(flags == other.flags) &&
(location == other.location) &&
(value == other.value);
}
void Token::setAtStartOfLine(bool start)
{
if (start)
flags |= AT_START_OF_LINE;
else
flags &= ~AT_START_OF_LINE;
}
void Token::setHasLeadingSpace(bool space)
{
if (space)
flags |= HAS_LEADING_SPACE;
else
flags &= ~HAS_LEADING_SPACE;
}
std::ostream& operator<<(std::ostream& out, const Token& token)
{
if (token.hasLeadingSpace())
......
src/compiler/preprocessor/new/Token.h
......@@ -50,35 +50,22 @@ struct Token
};
enum Flags
{
HAS_LEADING_SPACE = 1 << 0
AT_START_OF_LINE = 1 << 0,
HAS_LEADING_SPACE = 1 << 1
};
Token() : type(0), flags(0) { }
void reset()
{
type = 0;
flags = 0;
location = SourceLocation();
value.clear();
}
void reset();
bool equals(const Token& other) const;
bool equals(const Token& other) const
{
return (type == other.type) &&
(flags == other.flags) &&
(location == other.location) &&
(value == other.value);
}
// Returns true if this is the first token on line.
// It disregards any leading whitespace.
bool atStartOfLine() const { return (flags & AT_START_OF_LINE) != 0; }
void setAtStartOfLine(bool start);
bool hasLeadingSpace() const { return (flags & HAS_LEADING_SPACE) != 0; }
void setHasLeadingSpace(bool space)
{
if (space)
flags |= HAS_LEADING_SPACE;
else
flags &= ~HAS_LEADING_SPACE;
}
void setHasLeadingSpace(bool space);
int type;
int flags;
......
src/compiler/preprocessor/new/Tokenizer.cpp
......@@ -533,18 +533,20 @@ typedef pp::SourceLocation YYLTYPE;
yyextra->lineStart = true; \
} while(0);
#define YY_USER_ACTION \
do { \
pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while (scanLoc->cIndex >= input->length(scanLoc->sIndex)) \
{ \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \
} \
yylloc->file = yyfileno; \
yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \
#define YY_USER_ACTION \
do \
{ \
pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while ((scanLoc->sIndex < input->count()) && \
(scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
{ \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \
} \
yylloc->file = yyfileno; \
yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \
} while(0);
#define YY_INPUT(buf, result, maxSize) \
......@@ -1115,13 +1117,28 @@ YY_RULE_SETUP
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(COMMENT):
{
// YY_USER_ACTION is not invoked for handling EOF.
// Set the location for EOF token manually.
pp::Input* input = &yyextra->input;
pp::Input::Location* scanLoc = &yyextra->scanLoc;
int sIndexMax = std::max(0, input->count() - 1);
if (scanLoc->sIndex != sIndexMax)
{
// We can only reach here if there are empty strings at the
// end of the input.
scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
yyfileno = sIndexMax; yylineno = 1;
}
yylloc->file = yyfileno;
yylloc->line = yylineno;
yylval->clear();
if (YY_START == COMMENT)
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno),
"");
}
yylval->clear();
yyterminate();
}
YY_BREAK
......@@ -2289,9 +2306,11 @@ void Tokenizer::lex(Token* token)
{
token->type = pplex(&token->value,&token->location,mHandle);
token->setAtStartOfLine(mContext.lineStart);
mContext.lineStart = token->type == '\n';
token->setHasLeadingSpace(mContext.leadingSpace);
mContext.leadingSpace = false;
mContext.lineStart = token->type == '\n';
}
bool Tokenizer::initScanner()
......
src/compiler/preprocessor/new/Tokenizer.l
......@@ -42,18 +42,20 @@ typedef pp::SourceLocation YYLTYPE;
yyextra->lineStart = true; \
} while(0);
#define YY_USER_ACTION \
do { \
pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while (scanLoc->cIndex >= input->length(scanLoc->sIndex)) \
{ \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \
} \
yylloc->file = yyfileno; \
yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \
#define YY_USER_ACTION \
do \
{ \
pp::Input* input = &yyextra->input; \
pp::Input::Location* scanLoc = &yyextra->scanLoc; \
while ((scanLoc->sIndex < input->count()) && \
(scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
{ \
scanLoc->cIndex -= input->length(scanLoc->sIndex++); \
++yyfileno; yylineno = 1; \
} \
yylloc->file = yyfileno; \
yylloc->line = yylineno; \
scanLoc->cIndex += yyleng; \
} while(0);
#define YY_INPUT(buf, result, maxSize) \
......@@ -236,13 +238,28 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
}
<*><<EOF>> {
// YY_USER_ACTION is not invoked for handling EOF.
// Set the location for EOF token manually.
pp::Input* input = &yyextra->input;
pp::Input::Location* scanLoc = &yyextra->scanLoc;
int sIndexMax = std::max(0, input->count() - 1);
if (scanLoc->sIndex != sIndexMax)
{
// We can only reach here if there are empty strings at the
// end of the input.
scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
yyfileno = sIndexMax; yylineno = 1;
}
yylloc->file = yyfileno;
yylloc->line = yylineno;
yylval->clear();
if (YY_START == COMMENT)
{
yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT,
pp::SourceLocation(yyfileno, yylineno),
"");
}
yylval->clear();
yyterminate();
}
......@@ -273,9 +290,11 @@ void Tokenizer::lex(Token* token)
{
token->type = yylex(&token->value, &token->location, mHandle);
token->setAtStartOfLine(mContext.lineStart);
mContext.lineStart = token->type == '\n';
token->setHasLeadingSpace(mContext.leadingSpace);
mContext.leadingSpace = false;
mContext.lineStart = token->type == '\n';
}
bool Tokenizer::initScanner()
......
tests/preprocessor_tests/location_test.cpp
......@@ -35,7 +35,7 @@ TEST_F(LocationTest, String0_Line1)
loc.line = 1;
SCOPED_TRACE("String0_Line1");
preprocess(1, &str, 0, loc);
preprocess(1, &str, NULL, loc);
}
TEST_F(LocationTest, String0_Line2)
......@@ -46,7 +46,7 @@ TEST_F(LocationTest, String0_Line2)
loc.line = 2;
SCOPED_TRACE("String0_Line2");
preprocess(1, &str, 0, loc);
preprocess(1, &str, NULL, loc);
}
TEST_F(LocationTest, String1_Line1)
......@@ -57,7 +57,7 @@ TEST_F(LocationTest, String1_Line1)
loc.line = 1;
SCOPED_TRACE("String1_Line1");
preprocess(2, str, 0, loc);
preprocess(2, str, NULL, loc);
}
TEST_F(LocationTest, String1_Line2)
......@@ -68,7 +68,7 @@ TEST_F(LocationTest, String1_Line2)
loc.line = 2;
SCOPED_TRACE("String1_Line2");
preprocess(2, str, 0, loc);
preprocess(2, str, NULL, loc);
}
TEST_F(LocationTest, NewlineInsideCommentCounted)
......@@ -79,14 +79,14 @@ TEST_F(LocationTest, NewlineInsideCommentCounted)
loc.line = 3;
SCOPED_TRACE("NewlineInsideCommentCounted");
preprocess(1, &str, 0, loc);
preprocess(1, &str, NULL, loc);
}
TEST_F(LocationTest, ErrorLocationAfterComment)
{
const char* str = "/*\n\n*/@";
ASSERT_TRUE(mPreprocessor.init(1, &str, 0));
ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::INVALID_CHARACTER,
pp::SourceLocation(0, 3),
"@"));
......@@ -106,7 +106,7 @@ TEST_F(LocationTest, TokenStraddlingTwoStrings)
loc.line = 1;
SCOPED_TRACE("TokenStraddlingTwoStrings");
preprocess(2, str, 0, loc);
preprocess(2, str, NULL, loc);
}
TEST_F(LocationTest, TokenStraddlingThreeStrings)
......@@ -117,7 +117,61 @@ TEST_F(LocationTest, TokenStraddlingThreeStrings)
loc.line = 1;
SCOPED_TRACE("TokenStraddlingThreeStrings");
preprocess(3, str, 0, loc);
preprocess(3, str, NULL, loc);
}
TEST_F(LocationTest, EndOfFileWithoutNewline)
{
const char* const str[] = {"foo"};
ASSERT_TRUE(mPreprocessor.init(1, str, NULL));
pp::Token token;
mPreprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_EQ("foo", token.value);
EXPECT_EQ(0, token.location.file);
EXPECT_EQ(1, token.location.line);
mPreprocessor.lex(&token);
EXPECT_EQ(pp::Token::LAST, token.type);
EXPECT_EQ(0, token.location.file);
EXPECT_EQ(1, token.location.line);
}
TEST_F(LocationTest, EndOfFileAfterNewline)
{
const char* const str[] = {"foo\n"};
ASSERT_TRUE(mPreprocessor.init(1, str, NULL));
pp::Token token;
mPreprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_EQ("foo", token.value);
EXPECT_EQ(0, token.location.file);
EXPECT_EQ(1, token.location.line);
mPreprocessor.lex(&token);
EXPECT_EQ(pp::Token::LAST, token.type);
EXPECT_EQ(0, token.location.file);
EXPECT_EQ(2, token.location.line);
}
TEST_F(LocationTest, EndOfFileAfterEmptyString)
{
const char* const str[] = {"foo\n", "\n", ""};
ASSERT_TRUE(mPreprocessor.init(3, str, NULL));
pp::Token token;
mPreprocessor.lex(&token);
EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
EXPECT_EQ("foo", token.value);
EXPECT_EQ(0, token.location.file);
EXPECT_EQ(1, token.location.line);
mPreprocessor.lex(&token);
EXPECT_EQ(pp::Token::LAST, token.type);
EXPECT_EQ(2, token.location.file);
EXPECT_EQ(1, token.location.line);
}
// TODO(alokp): Add tests for #line directives.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment