Skip to content

A
angle
Commit dc0fa46a by Corentin Wallez Committed by Commit Bot
Options

preprocessor: Check for line number overflow

Also remove dead code in Tokenizer.l BUG=chromium:668842 Change-Id: Ice18313a64f0bb2242299993bfaa882a6578ad54 Reviewed-on: https://chromium-review.googlesource.com/435042Reviewed-by: 's avatarJamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarGeoff Lang <geofflang@chromium.org> Commit-Queue: Corentin Wallez <cwallez@chromium.org>
parent 80cdc376
Showing with 176 additions and 106 deletions
src/compiler/preprocessor/DiagnosticsBase.h
...@@ -63,6 +63,7 @@ class Diagnostics ...@@ -63,6 +63,7 @@ class Diagnostics
PP_INVALID_LINE_DIRECTIVE, PP_INVALID_LINE_DIRECTIVE,
PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL3, PP_NON_PP_TOKEN_BEFORE_EXTENSION_ESSL3,
PP_UNDEFINED_SHIFT, PP_UNDEFINED_SHIFT,
PP_TOKENIZER_ERROR,
PP_ERROR_END, PP_ERROR_END,
PP_WARNING_BEGIN, PP_WARNING_BEGIN,
......
src/compiler/preprocessor/Token.h
...@@ -19,7 +19,9 @@ struct Token ...@@ -19,7 +19,9 @@ struct Token
{ {
enum Type enum Type
{ {
LAST = 0, // EOF. // Calling this ERROR causes a conflict with wingdi.h
GOT_ERROR = -1,
LAST = 0, // EOF.
IDENTIFIER = 258, IDENTIFIER = 258,
......
src/compiler/preprocessor/Tokenizer.cpp
...@@ -724,8 +724,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner ); ...@@ -724,8 +724,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner );
yyg->yy_hold_char = *yy_cp; \ yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \ *yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp; yyg->yy_c_buf_p = yy_cp;
#define YY_NUM_RULES 38 #define YY_NUM_RULES 37
#define YY_END_OF_BUFFER 39 #define YY_END_OF_BUFFER 38
/* This struct is not used in this scanner, /* This struct is not used in this scanner,
but its presence is necessary. */ but its presence is necessary. */
struct yy_trans_info struct yy_trans_info
...@@ -733,18 +733,18 @@ struct yy_trans_info ...@@ -733,18 +733,18 @@ struct yy_trans_info
flex_int32_t yy_verify; flex_int32_t yy_verify;
flex_int32_t yy_nxt; flex_int32_t yy_nxt;
}; };
static const flex_int16_t yy_accept[98] = static const flex_int16_t yy_accept[95] =
{ 0, { 0,
0, 0, 0, 0, 39, 37, 34, 35, 35, 33, 0, 0, 0, 0, 38, 36, 34, 35, 35, 33,
7, 33, 33, 33, 33, 33, 33, 33, 33, 9, 7, 33, 33, 33, 33, 33, 33, 33, 33, 9,
9, 33, 33, 33, 8, 37, 33, 33, 3, 5, 9, 33, 33, 33, 8, 33, 33, 3, 5, 5,
5, 4, 34, 35, 19, 27, 20, 30, 25, 12, 4, 34, 35, 19, 27, 20, 30, 25, 12, 23,
23, 13, 24, 10, 2, 1, 26, 10, 9, 11, 13, 24, 10, 2, 1, 26, 10, 9, 11, 11,
11, 11, 9, 11, 9, 9, 14, 16, 18, 17, 11, 9, 11, 9, 9, 14, 16, 18, 17, 15,
15, 8, 36, 36, 31, 21, 32, 22, 3, 5, 8, 31, 21, 32, 22, 3, 5, 6, 11, 10,
6, 11, 10, 11, 10, 1, 10, 11, 10, 0, 11, 10, 1, 10, 11, 10, 0, 10, 9, 9,
10, 9, 9, 9, 28, 29, 0, 10, 10, 10, 9, 28, 29, 0, 10, 10, 10, 10, 9, 10,
10, 9, 10, 10, 9, 10, 0 10, 9, 10, 0
} ; } ;
...@@ -759,11 +759,11 @@ static const YY_CHAR yy_ec[256] = ...@@ -759,11 +759,11 @@ static const YY_CHAR yy_ec[256] =
19, 20, 9, 1, 21, 21, 21, 21, 22, 23, 19, 20, 9, 1, 21, 21, 21, 21, 22, 23,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 25, 24, 24, 26, 24, 24, 24, 24, 24, 24, 25, 24, 24, 26, 24, 24,
9, 27, 9, 28, 24, 1, 21, 21, 21, 21, 9, 1, 9, 27, 24, 1, 21, 21, 21, 21,
22, 23, 24, 24, 24, 24, 24, 24, 24, 24, 22, 23, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 25, 24, 24, 26, 24, 24, 24, 24, 24, 24, 25, 24, 24, 26,
24, 24, 9, 29, 9, 9, 1, 1, 1, 1, 24, 24, 9, 28, 9, 9, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
...@@ -780,101 +780,99 @@ static const YY_CHAR yy_ec[256] = ...@@ -780,101 +780,99 @@ static const YY_CHAR yy_ec[256] =
1, 1, 1, 1, 1 1, 1, 1, 1, 1
} ; } ;
static const YY_CHAR yy_meta[30] = static const YY_CHAR yy_meta[29] =
{ 0, { 0,
1, 1, 2, 2, 1, 1, 1, 1, 1, 3, 1, 1, 2, 2, 1, 1, 1, 1, 1, 3,
1, 1, 4, 1, 5, 5, 5, 1, 1, 1, 1, 1, 4, 1, 5, 5, 5, 1, 1, 1,
5, 5, 5, 5, 5, 5, 1, 1, 1 5, 5, 5, 5, 5, 5, 1, 1
} ; } ;
static const flex_int16_t yy_base[103] = static const flex_int16_t yy_base[100] =
{ 0, { 0,
0, 0, 27, 29, 137, 194, 133, 194, 117, 100, 0, 0, 26, 28, 133, 195, 130, 195, 128, 105,
194, 98, 26, 194, 94, 24, 28, 33, 32, 39, 195, 104, 25, 195, 100, 23, 27, 32, 31, 38,
51, 39, 80, 50, 0, 68, 25, 54, 0, 194, 50, 38, 93, 49, 0, 16, 51, 0, 195, 105,
88, 71, 80, 194, 194, 194, 194, 194, 194, 194, 87, 93, 195, 195, 195, 195, 195, 195, 195, 195,
194, 194, 194, 71, 194, 0, 194, 85, 55, 64, 195, 195, 67, 195, 0, 195, 81, 55, 84, 98,
99, 111, 53, 105, 0, 50, 55, 194, 194, 194, 110, 53, 61, 0, 52, 39, 195, 195, 195, 33,
40, 0, 194, 38, 194, 194, 194, 194, 0, 194, 0, 195, 195, 195, 195, 0, 195, 195, 113, 0,
194, 117, 0, 130, 0, 0, 0, 137, 0, 88, 126, 0, 0, 0, 133, 0, 56, 128, 0, 133,
113, 0, 131, 0, 194, 194, 143, 139, 152, 150, 0, 195, 195, 101, 141, 143, 145, 0, 15, 154,
0, 13, 153, 194, 0, 194, 194, 176, 31, 181, 195, 0, 195, 195, 177, 32, 182, 187, 189
186, 188
} ; } ;
static const flex_int16_t yy_def[103] = static const flex_int16_t yy_def[100] =
{ 0, { 0,
97, 1, 98, 98, 97, 97, 97, 97, 97, 97, 94, 1, 95, 95, 94, 94, 94, 94, 94, 94,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
20, 97, 97, 97, 99, 97, 97, 97, 100, 97, 20, 94, 94, 94, 96, 94, 94, 97, 94, 94,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
97, 97, 97, 97, 97, 101, 97, 97, 20, 20, 94, 94, 94, 94, 98, 94, 94, 20, 20, 49,
50, 51, 51, 102, 21, 51, 97, 97, 97, 97, 50, 50, 99, 21, 50, 94, 94, 94, 94, 94,
97, 99, 97, 97, 97, 97, 97, 97, 100, 97, 96, 94, 94, 94, 94, 97, 94, 94, 43, 43,
97, 44, 44, 72, 72, 101, 48, 51, 51, 97, 69, 69, 98, 47, 50, 50, 94, 51, 50, 99,
52, 51, 102, 51, 97, 97, 97, 74, 78, 97, 50, 94, 94, 94, 71, 75, 94, 50, 50, 94,
51, 51, 97, 97, 51, 97, 0, 97, 97, 97, 94, 50, 94, 0, 94, 94, 94, 94, 94
97, 97
} ; } ;
static const flex_int16_t yy_nxt[224] = static const flex_int16_t yy_nxt[224] =
{ 0, { 0,
6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 21, 22, 23, 24, 16, 17, 18, 19, 20, 21, 21, 22, 23, 24,
25, 25, 25, 25, 25, 25, 26, 27, 28, 30, 25, 25, 25, 25, 25, 25, 26, 27, 29, 30,
31, 30, 31, 37, 40, 62, 32, 95, 32, 42, 29, 30, 36, 39, 62, 31, 61, 31, 41, 92,
63, 45, 41, 65, 38, 46, 43, 44, 44, 44, 44, 40, 63, 37, 45, 42, 43, 43, 43, 46,
47, 48, 66, 49, 49, 50, 57, 58, 86, 51, 47, 83, 48, 48, 49, 56, 57, 82, 50, 51,
52, 51, 51, 53, 54, 55, 55, 55, 60, 61, 50, 50, 52, 53, 54, 54, 54, 59, 60, 64,
63, 64, 67, 85, 84, 56, 51, 82, 50, 50, 87, 87, 87, 50, 55, 50, 81, 79, 65, 69,
51, 33, 68, 72, 71, 73, 73, 73, 51, 51, 50, 70, 70, 70, 50, 50, 50, 69, 71, 72,
70, 72, 74, 75, 72, 72, 72, 51, 59, 77, 69, 69, 69, 50, 32, 74, 74, 74, 49, 49,
77, 77, 90, 90, 90, 51, 78, 79, 51, 51, 68, 50, 75, 76, 50, 50, 50, 67, 50, 50,
51, 51, 39, 51, 51, 51, 36, 51, 35, 34, 50, 58, 50, 50, 50, 90, 90, 90, 38, 50,
51, 80, 80, 97, 97, 81, 81, 81, 51, 51, 77, 77, 35, 34, 78, 78, 78, 69, 69, 69,
51, 72, 72, 72, 33, 91, 97, 97, 72, 72, 33, 32, 94, 94, 69, 69, 84, 84, 94, 94,
87, 87, 97, 51, 88, 88, 88, 87, 87, 97, 85, 85, 85, 84, 84, 50, 94, 86, 86, 86,
97, 89, 89, 89, 51, 92, 51, 93, 93, 93, 88, 94, 94, 94, 94, 94, 50, 89, 50, 87,
97, 75, 97, 97, 90, 90, 90, 93, 93, 93, 87, 87, 94, 72, 94, 76, 94, 91, 90, 90,
97, 97, 94, 97, 79, 96, 29, 29, 29, 29, 90, 94, 94, 94, 94, 94, 93, 28, 28, 28,
29, 69, 97, 97, 69, 69, 76, 97, 76, 76, 28, 28, 66, 94, 94, 66, 66, 73, 94, 73,
76, 83, 83, 5, 97, 97, 97, 97, 97, 97, 73, 73, 80, 80, 5, 94, 94, 94, 94, 94,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
97, 97, 97 94, 94, 94
} ; } ;
static const flex_int16_t yy_chk[224] = static const flex_int16_t yy_chk[224] =
{ 0, { 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3,
3, 4, 4, 13, 16, 99, 3, 92, 4, 17, 4, 4, 13, 16, 26, 3, 96, 4, 17, 89,
64, 19, 16, 27, 13, 19, 17, 18, 18, 18, 19, 16, 26, 13, 19, 17, 18, 18, 18, 19,
19, 20, 27, 20, 20, 20, 22, 22, 61, 20, 20, 60, 20, 20, 20, 22, 22, 56, 20, 20,
20, 20, 20, 20, 20, 21, 21, 21, 24, 24, 20, 20, 20, 20, 21, 21, 21, 24, 24, 27,
26, 26, 28, 57, 56, 21, 21, 53, 50, 50, 77, 77, 77, 53, 21, 21, 55, 52, 27, 43,
49, 33, 28, 44, 32, 44, 44, 44, 50, 50, 48, 43, 43, 43, 53, 53, 53, 43, 43, 43,
31, 44, 44, 44, 44, 44, 44, 48, 23, 48, 43, 43, 43, 47, 32, 47, 47, 47, 49, 49,
48, 48, 80, 80, 80, 48, 48, 48, 48, 48, 31, 47, 47, 47, 47, 47, 47, 30, 49, 49,
48, 51, 15, 51, 51, 51, 12, 54, 10, 9, 50, 23, 50, 50, 50, 84, 84, 84, 15, 50,
51, 52, 52, 81, 81, 52, 52, 52, 54, 54, 51, 51, 12, 10, 51, 51, 51, 69, 69, 69,
54, 72, 72, 72, 7, 81, 5, 0, 72, 72, 9, 7, 5, 0, 69, 69, 71, 71, 78, 78,
74, 74, 0, 83, 74, 74, 74, 78, 78, 88, 71, 71, 71, 75, 75, 80, 0, 75, 75, 75,
88, 78, 78, 78, 83, 83, 83, 87, 87, 87, 78, 85, 85, 86, 86, 0, 80, 80, 80, 87,
0, 88, 89, 89, 90, 90, 90, 93, 93, 93, 87, 87, 0, 85, 0, 86, 0, 87, 90, 90,
0, 0, 90, 0, 89, 93, 98, 98, 98, 98, 90, 0, 0, 0, 0, 0, 90, 95, 95, 95,
98, 100, 0, 0, 100, 100, 101, 0, 101, 101, 95, 95, 97, 0, 0, 97, 97, 98, 0, 98,
101, 102, 102, 97, 97, 97, 97, 97, 97, 97, 98, 98, 99, 99, 94, 94, 94, 94, 94, 94,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 94, 94, 94, 94, 94, 94, 94, 94, 94, 94,
97, 97, 97 94, 94, 94
} ; } ;
/* The intent behind this definition is that it'll catch /* The intent behind this definition is that it'll catch
...@@ -1378,13 +1376,13 @@ yy_match: ...@@ -1378,13 +1376,13 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{ {
yy_current_state = (int) yy_def[yy_current_state]; yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 98 ) if ( yy_current_state >= 95 )
yy_c = yy_meta[yy_c]; yy_c = yy_meta[yy_c];
} }
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
++yy_cp; ++yy_cp;
} }
while ( yy_current_state != 97 ); while ( yy_current_state != 94 );
yy_cp = yyg->yy_last_accepting_cpos; yy_cp = yyg->yy_last_accepting_cpos;
yy_current_state = yyg->yy_last_accepting_state; yy_current_state = yyg->yy_last_accepting_state;
...@@ -1429,7 +1427,14 @@ YY_RULE_SETUP ...@@ -1429,7 +1427,14 @@ YY_RULE_SETUP
case 5: case 5:
/* rule 5 can match eol */ /* rule 5 can match eol */
YY_RULE_SETUP YY_RULE_SETUP
{ ++yylineno; } {
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno;
}
YY_BREAK YY_BREAK
case 6: case 6:
YY_RULE_SETUP YY_RULE_SETUP
...@@ -1638,17 +1643,17 @@ case 35: ...@@ -1638,17 +1643,17 @@ case 35:
/* rule 35 can match eol */ /* rule 35 can match eol */
YY_RULE_SETUP YY_RULE_SETUP
{ {
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno; ++yylineno;
yylval->assign(1, '\n'); yylval->assign(1, '\n');
return '\n'; return '\n';
} }
YY_BREAK YY_BREAK
case 36: case 36:
/* rule 36 can match eol */
YY_RULE_SETUP
{ ++yylineno; }
YY_BREAK
case 37:
YY_RULE_SETUP YY_RULE_SETUP
{ {
yylval->assign(1, yytext[0]); yylval->assign(1, yytext[0]);
...@@ -1684,7 +1689,7 @@ case YY_STATE_EOF(COMMENT): ...@@ -1684,7 +1689,7 @@ case YY_STATE_EOF(COMMENT):
yyterminate(); yyterminate();
} }
YY_BREAK YY_BREAK
case 38: case 37:
YY_RULE_SETUP YY_RULE_SETUP
ECHO; ECHO;
YY_BREAK YY_BREAK
...@@ -1992,7 +1997,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) ...@@ -1992,7 +1997,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{ {
yy_current_state = (int) yy_def[yy_current_state]; yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 98 ) if ( yy_current_state >= 95 )
yy_c = yy_meta[yy_c]; yy_c = yy_meta[yy_c];
} }
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
...@@ -2022,11 +2027,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) ...@@ -2022,11 +2027,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{ {
yy_current_state = (int) yy_def[yy_current_state]; yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 98 ) if ( yy_current_state >= 95 )
yy_c = yy_meta[yy_c]; yy_c = yy_meta[yy_c];
} }
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
yy_is_jam = (yy_current_state == 97); yy_is_jam = (yy_current_state == 94);
(void)yyg; (void)yyg;
return yy_is_jam ? 0 : yy_current_state; return yy_is_jam ? 0 : yy_current_state;
...@@ -3006,7 +3011,18 @@ void Tokenizer::setMaxTokenSize(size_t maxTokenSize) ...@@ -3006,7 +3011,18 @@ void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
void Tokenizer::lex(Token *token) void Tokenizer::lex(Token *token)
{ {
token->type = yylex(&token->text, &token->location, mHandle); int tokenType = yylex(&token->text, &token->location, mHandle);
if (tokenType == Token::GOT_ERROR)
{
mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
token->type = Token::LAST;
}
else
{
token->type = tokenType;
}
if (token->text.size() > mMaxTokenSize) if (token->text.size() > mMaxTokenSize)
{ {
mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG, mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
......
src/compiler/preprocessor/Tokenizer.l
...@@ -110,7 +110,14 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -110,7 +110,14 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
"/*" { BEGIN(COMMENT); } "/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+ <COMMENT>[^*\r\n]+
<COMMENT>"*" <COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; } <COMMENT>{NEWLINE} {
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno;
}
<COMMENT>"*/" { <COMMENT>"*/" {
yyextra->leadingSpace = true; yyextra->leadingSpace = true;
BEGIN(INITIAL); BEGIN(INITIAL);
...@@ -237,13 +244,16 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") ...@@ -237,13 +244,16 @@ FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
[ \t\v\f]+ { yyextra->leadingSpace = true; } [ \t\v\f]+ { yyextra->leadingSpace = true; }
{NEWLINE} { {NEWLINE} {
if (yylineno == INT_MAX)
{
*yylval = "Integer overflow on line number";
return pp::Token::GOT_ERROR;
}
++yylineno; ++yylineno;
yylval->assign(1, '\n'); yylval->assign(1, '\n');
return '\n'; return '\n';
} }
\\{NEWLINE} { ++yylineno; }
. { . {
yylval->assign(1, yytext[0]); yylval->assign(1, yytext[0]);
return pp::Token::PP_OTHER; return pp::Token::PP_OTHER;
...@@ -318,7 +328,18 @@ void Tokenizer::setMaxTokenSize(size_t maxTokenSize) ...@@ -318,7 +328,18 @@ void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
void Tokenizer::lex(Token *token) void Tokenizer::lex(Token *token)
{ {
token->type = yylex(&token->text, &token->location, mHandle); int tokenType = yylex(&token->text, &token->location, mHandle);
if (tokenType == Token::GOT_ERROR)
{
mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
token->type = Token::LAST;
}
else
{
token->type = tokenType;
}
if (token->text.size() > mMaxTokenSize) if (token->text.size() > mMaxTokenSize)
{ {
mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG, mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
......
src/tests/preprocessor_tests/location_test.cpp
...@@ -267,6 +267,36 @@ TEST_F(LocationTest, LineDirectiveMissingNewline) ...@@ -267,6 +267,36 @@ TEST_F(LocationTest, LineDirectiveMissingNewline)
mPreprocessor.lex(&token); mPreprocessor.lex(&token);
} }
// Test for an error being generated when the line number overflows - regular version
TEST_F(LocationTest, LineOverflowRegular)
{
const char *str = "#line 2147483647\n\n";
ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
using testing::_;
// Error reported about EOF.
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_TOKENIZER_ERROR, _, _));
pp::Token token;
mPreprocessor.lex(&token);
}
// Test for an error being generated when the line number overflows - inside /* */ comment version
TEST_F(LocationTest, LineOverflowInComment)
{
const char *str = "#line 2147483647\n/*\n*/";
ASSERT_TRUE(mPreprocessor.init(1, &str, NULL));
using testing::_;
// Error reported about EOF.
EXPECT_CALL(mDiagnostics, print(pp::Diagnostics::PP_TOKENIZER_ERROR, _, _));
pp::Token token;
mPreprocessor.lex(&token);
}
struct LineTestParam struct LineTestParam
{ {
const char* str; const char* str;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment