Skip to content

A
angle
Commit dc0fa46a by Corentin Wallez Committed by Commit Bot
Options

preprocessor: Check for line number overflow

Also remove dead code in Tokenizer.l BUG=chromium:668842 Change-Id: Ice18313a64f0bb2242299993bfaa882a6578ad54 Reviewed-on: https://chromium-review.googlesource.com/435042Reviewed-by: 's avatarJamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarGeoff Lang <geofflang@chromium.org> Commit-Queue: Corentin Wallez <cwallez@chromium.org>
parent 80cdc376
Showing with 176 additions and 106 deletions
src/compiler/preprocessor/DiagnosticsBase.h
......@@ -63,6 +63,7 @@ class Diagnostics
PP_INVALID_LINE_DIRECTIVE,
PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL3,
PP_UNDEFINED_SHIFT,
PP_TOKENIZER_ERROR,
PP_ERROR_END,
PP_WARNING_BEGIN,
......
src/compiler/preprocessor/Token.h
......@@ -19,7 +19,9 @@ struct Token
{
enum Type
{
LAST = 0, // EOF.
// Calling this ERROR causes a conflict with wingdi.h
GOT_ERROR = -1,
LAST = 0, // EOF.
IDENTIFIER = 258,
......
src/compiler/preprocessor/Tokenizer.cpp
......@@ -724,8 +724,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner );
yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
#define YY_NUM_RULES 38
#define YY_END_OF_BUFFER 39
#define YY_NUM_RULES 37
#define YY_END_OF_BUFFER 38
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
......@@ -733,18 +733,18 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
static const flex_int16_t yy_accept[98] =
static const flex_int16_t yy_accept[95] =
{ 0,
0, 0, 0, 0, 39, 37, 34, 35, 35, 33,
0, 0, 0, 0, 38, 36, 34, 35, 35, 33,
7, 33, 33, 33, 33, 33, 33, 33, 33, 9,
9, 33, 33, 33, 8, 37, 33, 33, 3, 5,
5, 4, 34, 35, 19, 27, 20, 30, 25, 12,
23, 13, 24, 10, 2, 1, 26, 10, 9, 11,
11, 11, 9, 11, 9, 9, 14, 16, 18, 17,
15, 8, 36, 36, 31, 21, 32, 22, 3, 5,
6, 11, 10, 11, 10, 1, 10, 11, 10, 0,
10, 9, 9, 9, 28, 29, 0, 10, 10, 10,
10, 9, 10, 10, 9, 10, 0
9, 33, 33, 33, 8, 33, 33, 3, 5, 5,
4, 34, 35, 19, 27, 20, 30, 25, 12, 23,
13, 24, 10, 2, 1, 26, 10, 9, 11, 11,
11, 9, 11, 9, 9, 14, 16, 18, 17, 15,
8, 31, 21, 32, 22, 3, 5, 6, 11, 10,
11, 10, 1, 10, 11, 10, 0, 10, 9, 9,
9, 28, 29, 0, 10, 10, 10, 10, 9, 10,
10, 9, 10, 0
} ;
......@@ -759,11 +759,11 @@ static const YY_CHAR yy_ec[256] =
19, 20, 9, 1, 21, 21, 21, 21, 22, 23,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 25, 24, 24, 26, 24, 24,
9, 27, 9, 28, 24, 1, 21, 21, 21, 21,
9, 1, 9, 27, 24, 1, 21, 21, 21, 21,
22, 23, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 25, 24, 24, 26,
24, 24, 9, 29, 9, 9, 1, 1, 1, 1,
24, 24, 9, 28, 9, 9, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
......@@ -780,101 +780,99 @@ static const YY_CHAR yy_ec[256] =
1, 1, 1, 1, 1
} ;
static const YY_CHAR yy_meta[30] =
static const YY_CHAR yy_meta[29] =
{ 0,
1, 1, 2, 2, 1, 1, 1, 1, 1, 3,
1, 1, 4, 1, 5, 5, 5, 1, 1, 1,
5, 5, 5, 5, 5, 5, 1, 1, 1
5, 5, 5, 5, 5, 5, 1, 1
} ;
static const flex_int16_t yy_base[103] =
static const flex_int16_t yy_base[100] =
{ 0,
0, 0, 27, 29, 137, 194, 133, 194, 117, 100,
194, 98, 26, 194, 94, 24, 28, 33, 32, 39,
51, 39, 80, 50, 0, 68, 25, 54, 0, 194,
88, 71, 80, 194, 194, 194, 194, 194, 194, 194,
194, 194, 194, 71, 194, 0, 194, 85, 55, 64,
99, 111, 53, 105, 0, 50, 55, 194, 194, 194,
40, 0, 194, 38, 194, 194, 194, 194, 0, 194,
194, 117, 0, 130, 0, 0, 0, 137, 0, 88,
113, 0, 131, 0, 194, 194, 143, 139, 152, 150,
0, 13, 153, 194, 0, 194, 194, 176, 31, 181,
186, 188
0, 0, 26, 28, 133, 195, 130, 195, 128, 105,
195, 104, 25, 195, 100, 23, 27, 32, 31, 38,
50, 38, 93, 49, 0, 16, 51, 0, 195, 105,
87, 93, 195, 195, 195, 195, 195, 195, 195, 195,
195, 195, 67, 195, 0, 195, 81, 55, 84, 98,
110, 53, 61, 0, 52, 39, 195, 195, 195, 33,
0, 195, 195, 195, 195, 0, 195, 195, 113, 0,
126, 0, 0, 0, 133, 0, 56, 128, 0, 133,
0, 195, 195, 101, 141, 143, 145, 0, 15, 154,
195, 0, 195, 195, 177, 32, 182, 187, 189
} ;
static const flex_int16_t yy_def[103] =
static const flex_int16_t yy_def[100] =
{ 0,
97, 1, 98, 98, 97, 97, 97, 97, 97, 97,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97,
20, 97, 97, 97, 99, 97, 97, 97, 100, 97,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97,
97, 97, 97, 97, 97, 101, 97, 97, 20, 20,
50, 51, 51, 102, 21, 51, 97, 97, 97, 97,
97, 99, 97, 97, 97, 97, 97, 97, 100, 97,
97, 44, 44, 72, 72, 101, 48, 51, 51, 97,
52, 51, 102, 51, 97, 97, 97, 74, 78, 97,
51, 51, 97, 97, 51, 97, 0, 97, 97, 97,
97, 97
94, 1, 95, 95, 94, 94, 94, 94, 94, 94,
94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
20, 94, 94, 94, 96, 94, 94, 97, 94, 94,
94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
94, 94, 94, 94, 98, 94, 94, 20, 20, 49,
50, 50, 99, 21, 50, 94, 94, 94, 94, 94,
96, 94, 94, 94, 94, 97, 94, 94, 43, 43,
69, 69, 98, 47, 50, 50, 94, 51, 50, 99,
50, 94, 94, 94, 71, 75, 94, 50, 50, 94,
94, 50, 94, 0, 94, 94, 94, 94, 94
} ;
static const flex_int16_t yy_nxt[224] =
{ 0,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 21, 22, 23, 24,
25, 25, 25, 25, 25, 25, 26, 27, 28, 30,
31, 30, 31, 37, 40, 62, 32, 95, 32, 42,
63, 45, 41, 65, 38, 46, 43, 44, 44, 44,
47, 48, 66, 49, 49, 50, 57, 58, 86, 51,
52, 51, 51, 53, 54, 55, 55, 55, 60, 61,
63, 64, 67, 85, 84, 56, 51, 82, 50, 50,
51, 33, 68, 72, 71, 73, 73, 73, 51, 51,
70, 72, 74, 75, 72, 72, 72, 51, 59, 77,
77, 77, 90, 90, 90, 51, 78, 79, 51, 51,
51, 51, 39, 51, 51, 51, 36, 51, 35, 34,
51, 80, 80, 97, 97, 81, 81, 81, 51, 51,
51, 72, 72, 72, 33, 91, 97, 97, 72, 72,
87, 87, 97, 51, 88, 88, 88, 87, 87, 97,
97, 89, 89, 89, 51, 92, 51, 93, 93, 93,
97, 75, 97, 97, 90, 90, 90, 93, 93, 93,
97, 97, 94, 97, 79, 96, 29, 29, 29, 29,
29, 69, 97, 97, 69, 69, 76, 97, 76, 76,
76, 83, 83, 5, 97, 97, 97, 97, 97, 97,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97,
97, 97, 97
25, 25, 25, 25, 25, 25, 26, 27, 29, 30,
29, 30, 36, 39, 62, 31, 61, 31, 41, 92,
44, 40, 63, 37, 45, 42, 43, 43, 43, 46,
47, 83, 48, 48, 49, 56, 57, 82, 50, 51,
50, 50, 52, 53, 54, 54, 54, 59, 60, 64,
87, 87, 87, 50, 55, 50, 81, 79, 65, 69,
50, 70, 70, 70, 50, 50, 50, 69, 71, 72,
69, 69, 69, 50, 32, 74, 74, 74, 49, 49,
68, 50, 75, 76, 50, 50, 50, 67, 50, 50,
50, 58, 50, 50, 50, 90, 90, 90, 38, 50,
77, 77, 35, 34, 78, 78, 78, 69, 69, 69,
33, 32, 94, 94, 69, 69, 84, 84, 94, 94,
85, 85, 85, 84, 84, 50, 94, 86, 86, 86,
88, 94, 94, 94, 94, 94, 50, 89, 50, 87,
87, 87, 94, 72, 94, 76, 94, 91, 90, 90,
90, 94, 94, 94, 94, 94, 93, 28, 28, 28,
28, 28, 66, 94, 94, 66, 66, 73, 94, 73,
73, 73, 80, 80, 5, 94, 94, 94, 94, 94,
94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
94, 94, 94
} ;
static const flex_int16_t yy_chk[224] =
{ 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 3,
3, 4, 4, 13, 16, 99, 3, 92, 4, 17,
64, 19, 16, 27, 13, 19, 17, 18, 18, 18,
19, 20, 27, 20, 20, 20, 22, 22, 61, 20,
20, 20, 20, 20, 20, 21, 21, 21, 24, 24,
26, 26, 28, 57, 56, 21, 21, 53, 50, 50,
49, 33, 28, 44, 32, 44, 44, 44, 50, 50,
31, 44, 44, 44, 44, 44, 44, 48, 23, 48,
48, 48, 80, 80, 80, 48, 48, 48, 48, 48,
48, 51, 15, 51, 51, 51, 12, 54, 10, 9,
51, 52, 52, 81, 81, 52, 52, 52, 54, 54,
54, 72, 72, 72, 7, 81, 5, 0, 72, 72,
74, 74, 0, 83, 74, 74, 74, 78, 78, 88,
88, 78, 78, 78, 83, 83, 83, 87, 87, 87,
0, 88, 89, 89, 90, 90, 90, 93, 93, 93,
0, 0, 90, 0, 89, 93, 98, 98, 98, 98,
98, 100, 0, 0, 100, 100, 101, 0, 101, 101,
101, 102, 102, 97, 97, 97, 97, 97, 97, 97,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97,
97, 97, 97
1, 1, 1, 1, 1, 1, 1, 1, 3, 3,
4, 4, 13, 16, 26, 3, 96, 4, 17, 89,
19, 16, 26, 13, 19, 17, 18, 18, 18, 19,
20, 60, 20, 20, 20, 22, 22, 56, 20, 20,
20, 20, 20, 20, 21, 21, 21, 24, 24, 27,
77, 77, 77, 53, 21, 21, 55, 52, 27, 43,
48, 43, 43, 43, 53, 53, 53, 43, 43, 43,
43, 43, 43, 47, 32, 47, 47, 47, 49, 49,
31, 47, 47, 47, 47, 47, 47, 30, 49, 49,
50, 23, 50, 50, 50, 84, 84, 84, 15, 50,
51, 51, 12, 10, 51, 51, 51, 69, 69, 69,
9, 7, 5, 0, 69, 69, 71, 71, 78, 78,
71, 71, 71, 75, 75, 80, 0, 75, 75, 75,
78, 85, 85, 86, 86, 0, 80, 80, 80, 87,
87, 87, 0, 85, 0, 86, 0, 87, 90, 90,
90, 0, 0, 0, 0, 0, 90, 95, 95, 95,
95, 95, 97, 0, 0, 97, 97, 98, 0, 98,
98, 98, 99, 99, 94, 94, 94, 94, 94, 94,
94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
94, 94, 94
} ;
/* The intent behind this definition is that it'll catch
......@@ -1378,13 +1376,13 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 98 )
if ( yy_current_state >= 95 )
yy_c = yy_meta[yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
++yy_cp;
}
while ( yy_current_state != 97 );
while ( yy_current_state != 94 );
yy_cp = yyg->yy_last_accepting_cpos;
yy_current_state = yyg->yy_last_accepting_state;
......@@ -1429,7 +1427,14 @@ YY_RULE_SETUP
case 5:
/* rule 5 can match eol */
YY_RULE_SETUP
{ ++yylineno; }
{
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno;
}
YY_BREAK
case 6:
YY_RULE_SETUP
......@@ -1638,17 +1643,17 @@ case 35:
/* rule 35 can match eol */
YY_RULE_SETUP
{
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno;
yylval->assign(1, '\n');
return '\n';
}
YY_BREAK
case 36:
/* rule 36 can match eol */
YY_RULE_SETUP
{ ++yylineno; }
YY_BREAK
case 37:
YY_RULE_SETUP
{
yylval->assign(1, yytext[0]);
......@@ -1684,7 +1689,7 @@ case YY_STATE_EOF(COMMENT):
yyterminate();
}
YY_BREAK
case 38:
case 37:
YY_RULE_SETUP
ECHO;
YY_BREAK
......@@ -1992,7 +1997,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 98 )
if ( yy_current_state >= 95 )
yy_c = yy_meta[yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
......@@ -2022,11 +2027,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 98 )
if ( yy_current_state >= 95 )
yy_c = yy_meta[yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
yy_is_jam = (yy_current_state == 97);
yy_is_jam = (yy_current_state == 94);
(void)yyg;
return yy_is_jam ? 0 : yy_current_state;
......@@ -3006,7 +3011,18 @@ void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
void Tokenizer::lex(Token *token)
{
token->type = yylex(&token->text, &token->location, mHandle);
int tokenType = yylex(&token->text, &token->location, mHandle);
if (tokenType == Token::GOT_ERROR)
{
mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
token->type = Token::LAST;
}
else
{
token->type = tokenType;
}
if (token->text.size() > mMaxTokenSize)
{
mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
......
src/compiler/preprocessor/Tokenizer.l
......@@ -110,7 +110,14 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>{NEWLINE} {
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno;
}
<COMMENT>"*/" {
yyextra->leadingSpace = true;
BEGIN(INITIAL);
......@@ -237,13 +244,16 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
[ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} {
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno;
yylval->assign(1, '\n');
return '\n';
}
\\{NEWLINE} { ++yylineno; }
. {
yylval->assign(1, yytext[0]);
return pp::Token::PP_OTHER;
......@@ -318,7 +328,18 @@ void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
void Tokenizer::lex(Token *token)
{
token->type = yylex(&token->text, &token->location, mHandle);
int tokenType = yylex(&token->text, &token->location, mHandle);
if (tokenType == Token::GOT_ERROR)
{
mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
token->type = Token::LAST;
}
else
{
token->type = tokenType;
}
if (token->text.size() > mMaxTokenSize)
{
mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
......
src/tests/preprocessor_tests/location_test.cpp
......@@ -267,6 +267,36 @@ TEST_F(LocationTest, LineDirectiveMissingNewline)
mPreprocessor.lex(&token);
}
// Test for an error being generated when the line number overflows - regular version
TEST_F(LocationTest, LineOverflowRegular)
{
const char *str = "#line 2147483647\n\n";
ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
using testing::_;
// Error reported about EOF.
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_TOKENIZER_ERROR, _, _));
pp::Token token;
mPreprocessor.lex(&token);
}
// Test for an error being generated when the line number overflows - inside /* */ comment version
TEST_F(LocationTest, LineOverflowInComment)
{
const char *str = "#line 2147483647\n/*\n*/";
ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
using testing::_;
// Error reported about EOF.
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_TOKENIZER_ERROR, _, _));
pp::Token token;
mPreprocessor.lex(&token);
}
struct LineTestParam
{
const char* str;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment